From fd47f3b973ec74130ef8a7cf3d8026df6eb8593e Mon Sep 17 00:00:00 2001 From: lipeng <734991033@qq.com> Date: Sun, 6 Apr 2025 21:21:16 +0800 Subject: [PATCH 1/3] =?UTF-8?q?excuter(cpu/cuda):reshape=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E4=B8=BA=E6=94=AF=E6=8C=81=E9=9D=9Einplace?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/deepx/tensorfunc/changeshape.hpp | 6 +++--- excuter/op-mem-cuda/src/deepx/tf/changeshape.hpp | 14 +++++++------- excuter/op-mem-ompsimd/src/client/tfs.cpp | 5 ++++- .../deepx/tensorfunc/changeshape_miaobyte.hpp | 14 +++++++++++--- .../op-mem-ompsimd/src/deepx/tf/changeshape.hpp | 16 ++++++++-------- 5 files changed, 33 insertions(+), 22 deletions(-) diff --git a/excuter/cpp-common/src/deepx/tensorfunc/changeshape.hpp b/excuter/cpp-common/src/deepx/tensorfunc/changeshape.hpp index c2eff6b1..1ed42cd6 100644 --- a/excuter/cpp-common/src/deepx/tensorfunc/changeshape.hpp +++ b/excuter/cpp-common/src/deepx/tensorfunc/changeshape.hpp @@ -11,14 +11,14 @@ namespace deepx::tensorfunc template struct reshapeDispatcher { - static void reshape(Tensor &tensor, const std::vector &new_shape) = delete; + static void reshape(const Tensor &tensor, const std::vector &new_shape,Tensor &output) = delete; }; // A.reshape(new_shape) template - void reshape(Tensor &tensor, const std::vector &new_shape) + void reshape(const Tensor &tensor, const std::vector &new_shape,Tensor &output) { - reshapeDispatcher::reshape(tensor, new_shape); + reshapeDispatcher::reshape(tensor, new_shape,output); } template diff --git a/excuter/op-mem-cuda/src/deepx/tf/changeshape.hpp b/excuter/op-mem-cuda/src/deepx/tf/changeshape.hpp index 68d03767..34046a1c 100644 --- a/excuter/op-mem-cuda/src/deepx/tf/changeshape.hpp +++ b/excuter/op-mem-cuda/src/deepx/tf/changeshape.hpp @@ -26,7 +26,7 @@ namespace deepx::tf string math_formula() const override { - return "T2=T1.reshape(shape)"; + return "T1.reshape(shape)->T2"; } shared_ptr clone() const override @@ -47,22 +47,22 @@ namespace deepx::tf switch (input_type) { case Precision::Float64: - reshape(*mem->gettensor(this->args[0].textvalue), shape); + reshape(*mem->gettensor(this->args[0].textvalue), shape, *mem->gettensor(this->returns[0].textvalue)); break; case Precision::Float32: - reshape(*mem->gettensor(this->args[0].textvalue), shape); + reshape(*mem->gettensor(this->args[0].textvalue), shape, *mem->gettensor(this->returns[0].textvalue)); break; case Precision::Int64: - reshape(*mem->gettensor(this->args[0].textvalue), shape); + reshape(*mem->gettensor(this->args[0].textvalue), shape, *mem->gettensor(this->returns[0].textvalue)); break; case Precision::Int32: - reshape(*mem->gettensor(this->args[0].textvalue), shape); + reshape(*mem->gettensor(this->args[0].textvalue), shape, *mem->gettensor(this->returns[0].textvalue)); break; case Precision::Int16: - reshape(*mem->gettensor(this->args[0].textvalue), shape); + reshape(*mem->gettensor(this->args[0].textvalue), shape, *mem->gettensor(this->returns[0].textvalue)); break; case Precision::Int8: - reshape(*mem->gettensor(this->args[0].textvalue), shape); + reshape(*mem->gettensor(this->args[0].textvalue), shape, *mem->gettensor(this->returns[0].textvalue)); break; default: error = "Unsupported type: " + precision_str(input_type); diff --git a/excuter/op-mem-ompsimd/src/client/tfs.cpp b/excuter/op-mem-ompsimd/src/client/tfs.cpp index 57e2c206..9c92fe74 100644 --- a/excuter/op-mem-ompsimd/src/client/tfs.cpp +++ b/excuter/op-mem-ompsimd/src/client/tfs.cpp @@ -328,7 +328,10 @@ namespace deepx::tf Param("A", DataCategory::Tensor, Precision::Any), Param("shape", DataCategory::Vector, Precision::Int32), }), - vector())); + vector( + { + Param("B", DataCategory::Tensor, Precision::Any), + }))); tffactory.add_tf(std::make_shared>(vector( { diff --git a/excuter/op-mem-ompsimd/src/deepx/tensorfunc/changeshape_miaobyte.hpp b/excuter/op-mem-ompsimd/src/deepx/tensorfunc/changeshape_miaobyte.hpp index c03f3c0d..2170bc59 100644 --- a/excuter/op-mem-ompsimd/src/deepx/tensorfunc/changeshape_miaobyte.hpp +++ b/excuter/op-mem-ompsimd/src/deepx/tensorfunc/changeshape_miaobyte.hpp @@ -15,9 +15,9 @@ namespace deepx::tensorfunc template struct reshapeDispatcher { - static void reshape(Tensor &tensor, const std::vector &shape) + static void reshape(const Tensor &tensor, const std::vector &shape,Tensor &output) { // 参数改为单个tensor引用 - + int new_prod = 1; for (int dim : shape) { @@ -28,7 +28,15 @@ namespace deepx::tensorfunc { throw std::invalid_argument("Shape size mismatch"); } - tensor.shape = Shape(shape); + if (tensor.data == output.data) + { + output.shape = Shape(shape); + } + else + { + output.shape = Shape(shape); + output.copyer(tensor.data, output.data, tensor.shape.size); + } } }; diff --git a/excuter/op-mem-ompsimd/src/deepx/tf/changeshape.hpp b/excuter/op-mem-ompsimd/src/deepx/tf/changeshape.hpp index fa125766..28fc4caf 100644 --- a/excuter/op-mem-ompsimd/src/deepx/tf/changeshape.hpp +++ b/excuter/op-mem-ompsimd/src/deepx/tf/changeshape.hpp @@ -25,7 +25,7 @@ namespace deepx::tf string math_formula() const override { - return "T2=T1.reshape(shape)"; + return "T1.reshape(shape)->T2"; } shared_ptr clone() const override @@ -46,22 +46,22 @@ namespace deepx::tf switch (input_type) { case Precision::Float64: - reshape(*mem->gettensor(this->args[0].textvalue), shape); + reshape(*mem->gettensor(this->args[0].textvalue), shape, *mem->gettensor(this->returns[0].textvalue)); break; case Precision::Float32: - reshape(*mem->gettensor(this->args[0].textvalue), shape); + reshape(*mem->gettensor(this->args[0].textvalue), shape, *mem->gettensor(this->returns[0].textvalue)); break; case Precision::Int64: - reshape(*mem->gettensor(this->args[0].textvalue), shape); + reshape(*mem->gettensor(this->args[0].textvalue), shape, *mem->gettensor(this->returns[0].textvalue)); break; case Precision::Int32: - reshape(*mem->gettensor(this->args[0].textvalue), shape); + reshape(*mem->gettensor(this->args[0].textvalue), shape, *mem->gettensor(this->returns[0].textvalue)); break; case Precision::Int16: - reshape(*mem->gettensor(this->args[0].textvalue), shape); + reshape(*mem->gettensor(this->args[0].textvalue), shape, *mem->gettensor(this->returns[0].textvalue)); break; case Precision::Int8: - reshape(*mem->gettensor(this->args[0].textvalue), shape); + reshape(*mem->gettensor(this->args[0].textvalue), shape, *mem->gettensor(this->returns[0].textvalue)); break; default: error = "Unsupported type: " + precision_str(input_type); @@ -85,7 +85,7 @@ namespace deepx::tf string math_formula() const override { - return "T2 = T1.transpose(dimorder=[1,0])"; + return "T1.transpose(dimorder=[1,0])->T2"; } shared_ptr clone() const override From a5012443970c034c5529cbe379fd8f205786fa0c Mon Sep 17 00:00:00 2001 From: lipeng <734991033@qq.com> Date: Sun, 6 Apr 2025 21:21:25 +0800 Subject: [PATCH 2/3] =?UTF-8?q?excuter(cpu/cuda):reshape=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E4=B8=BA=E6=94=AF=E6=8C=81=E9=9D=9Einplace?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- excuter/op-mem-cuda/src/client/tfs.cpp | 5 +- .../deepx/tensorfunc/changeshape_miaobyte.hpp | 84 +++++++++++-------- 2 files changed, 53 insertions(+), 36 deletions(-) diff --git a/excuter/op-mem-cuda/src/client/tfs.cpp b/excuter/op-mem-cuda/src/client/tfs.cpp index c0d9bd9c..07368d1b 100644 --- a/excuter/op-mem-cuda/src/client/tfs.cpp +++ b/excuter/op-mem-cuda/src/client/tfs.cpp @@ -337,7 +337,10 @@ namespace deepx::tf Param("A", DataCategory::Tensor, Precision::Any), Param("shape", DataCategory::Vector, Precision::Int32), }), - vector())); + vector( + { + Param("B", DataCategory::Tensor, Precision::Any), + }))); tffactory.add_tf(std::make_shared>(vector( { diff --git a/excuter/op-mem-cuda/src/deepx/tensorfunc/changeshape_miaobyte.hpp b/excuter/op-mem-cuda/src/deepx/tensorfunc/changeshape_miaobyte.hpp index 4c9f5c4d..3c879f85 100644 --- a/excuter/op-mem-cuda/src/deepx/tensorfunc/changeshape_miaobyte.hpp +++ b/excuter/op-mem-cuda/src/deepx/tensorfunc/changeshape_miaobyte.hpp @@ -14,13 +14,27 @@ namespace deepx::tensorfunc template struct reshapeDispatcher { - static void reshape(Tensor &tensor, const std::vector &new_shape) + static void reshape(const Tensor &tensor, const std::vector &shape, Tensor &output) { - if (tensor.shape.dim != new_shape.size()) + int new_prod = 1; + for (int dim : shape) { - throw std::runtime_error("Tensor shapes must match for reshape"); + new_prod *= dim; + } + + if (tensor.shape.size != new_prod) + { + throw std::invalid_argument("Shape size mismatch"); + } + if (tensor.data == output.data) + { + output.shape = Shape(shape); + } + else + { + output.shape = Shape(shape); + output.copyer(tensor.data, output.data, tensor.shape.size); } - tensor.shape = Shape(new_shape); } }; @@ -35,46 +49,46 @@ namespace deepx::tensorfunc } auto [actual_blocks, optimal_block_size] = BestDims(tensor.shape.size); launch_transpose(actual_blocks, optimal_block_size, - tensor.data, tensor.shape.strides.data(), - output.data, output.shape.strides.data(), - tensor.shape.dim, tensor.shape.size, dim_order.data()); + tensor.data, tensor.shape.strides.data(), + output.data, output.shape.strides.data(), + tensor.shape.dim, tensor.shape.size, dim_order.data()); } }; template struct concatDispatcher { - static void concat(const vector*> tensors, const int axis, Tensor &C) - { - //checkshape - if (!checkShapeConcat(tensors, axis, C)) - { - throw TensorShapeError("Output tensor shape size must match the sum of input tensor shape sizes for concat"); - } + static void concat(const vector *> tensors, const int axis, Tensor &C) + { + // checkshape + if (!checkShapeConcat(tensors, axis, C)) + { + throw TensorShapeError("Output tensor shape size must match the sum of input tensor shape sizes for concat"); + } + + vector tensorsData(tensors.size()); + for (int i = 0; i < tensors.size(); i++) + { + tensorsData[i] = tensors[i]->data; + } - vector tensorsData(tensors.size()); - for (int i = 0; i < tensors.size(); i++) - { - tensorsData[i] = tensors[i]->data; - } + vector inputStrides; + for (int i = 0; i < tensors.size(); i++) + { + std::copy(tensors[i]->shape.strides.data(), tensors[i]->shape.strides.data() + tensors[i]->shape.dim, std::back_inserter(inputStrides)); + } - vector< int> inputStrides; - for (int i = 0; i < tensors.size(); i++) - { - std::copy(tensors[i]->shape.strides.data(), tensors[i]->shape.strides.data() + tensors[i]->shape.dim, std::back_inserter(inputStrides)); - } - - vector shapeAtAxis(tensors.size()); - for (int i = 0; i < tensors.size(); i++) - { - shapeAtAxis[i] = tensors[i]->shape[axis]; - } + vector shapeAtAxis(tensors.size()); + for (int i = 0; i < tensors.size(); i++) + { + shapeAtAxis[i] = tensors[i]->shape[axis]; + } - launch_concat(tensorsData.data(), inputStrides.data(), - C.data, C.shape.strides.data(), - C.shape.dim, - C.shape.size, - axis, tensors.size(), shapeAtAxis.data()); + launch_concat(tensorsData.data(), inputStrides.data(), + C.data, C.shape.strides.data(), + C.shape.dim, + C.shape.size, + axis, tensors.size(), shapeAtAxis.data()); }; }; } From b9e7dc4aaaf46ddb7b98c5983c690e5f833f6e2c Mon Sep 17 00:00:00 2001 From: lipeng <734991033@qq.com> Date: Sun, 6 Apr 2025 23:30:51 +0800 Subject: [PATCH 3/3] =?UTF-8?q?front&excuter:=E8=81=94=E5=90=88=E8=B0=83?= =?UTF-8?q?=E8=AF=95reshape?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- doc/excuter/op-mem-cuda/list.md | 2 +- excuter/cpp-common/src/deepx/tf/tf.hpp | 2 +- .../deepx/tensorfunc/changeshape_miaobyte.hpp | 7 +- .../deepx/tensorfunc/changeshape_miaobyte.hpp | 7 +- front/py/deepx/nn/deepxir.py | 4 +- front/py/deepx/nn/functional/__init__.py | 4 +- front/py/deepx/nn/functional/changeshape.py | 279 +++++++++--------- front/py/deepx/nn/functional/elementwise.py | 2 +- front/py/deepx/optim/optimizer.py | 6 +- front/py/deepx/optim/sgd.py | 13 + front/py/deepx/tensor/changeshape.py | 53 ++-- .../examples/2_ir/4_changeshape_reshape.dot | 33 +++ .../2_ir/4_changeshape_reshape.dot.svg | 146 +++++++++ 13 files changed, 381 insertions(+), 177 deletions(-) create mode 100644 front/py/examples/2_ir/4_changeshape_reshape.dot create mode 100644 front/py/examples/2_ir/4_changeshape_reshape.dot.svg diff --git a/doc/excuter/op-mem-cuda/list.md b/doc/excuter/op-mem-cuda/list.md index 09937bd1..80886ea9 100644 --- a/doc/excuter/op-mem-cuda/list.md +++ b/doc/excuter/op-mem-cuda/list.md @@ -6,7 +6,7 @@ |-----------|--------|------------|--------------|----------------| | concat | miaobyte | concat(listtensor tensors, var axis)->(tensor result) | Tresult = concat([T1, T2...], axis=3) | concat(listtensor tensors, var axis)->(tensor result) | | transpose | miaobyte | transpose(tensor A, vector dim_order)->(tensor C) | T2 = T1.transpose(dimorder=[1,0]) | transpose(tensor A, vector dim_order)->(tensor C) | -| reshape | miaobyte | reshape(tensor A, vector shape)->() | T2=T1.reshape(shape) | reshape(tensor A, vector shape)->() | +| reshape | miaobyte | reshape(tensor A, vector shape)->(tensor B) | T1.reshape(shape)->T2 | reshape(tensor A, vector shape)->(tensor B) | | matmul | cublas | matmul(tensor A, tensor B)->(tensor C) | T3=T1 @ T2 | matmul(tensor A, tensor B)->(tensor C) | | comparescalar | miaobyte | comparescalar(tensor A, var scalar)->(tensor mask) | mask=compare(T1, scalar) | comparescalar(tensor A, var scalar)->(tensor mask) | | add | cublas | add(tensor a, tensor b)->(tensor c) | T3=T1+T2 | add(tensor a, tensor b)->(tensor c) | diff --git a/excuter/cpp-common/src/deepx/tf/tf.hpp b/excuter/cpp-common/src/deepx/tf/tf.hpp index 90c0d3cb..f5b17204 100644 --- a/excuter/cpp-common/src/deepx/tf/tf.hpp +++ b/excuter/cpp-common/src/deepx/tf/tf.hpp @@ -110,7 +110,7 @@ namespace deepx::tf } std::stringstream ss(textvalue); std::string item; - while (std::getline(ss, item, ',')) + while (std::getline(ss, item, ' ')) { result.push_back(to(item)); } diff --git a/excuter/op-mem-cuda/src/deepx/tensorfunc/changeshape_miaobyte.hpp b/excuter/op-mem-cuda/src/deepx/tensorfunc/changeshape_miaobyte.hpp index 3c879f85..fb972636 100644 --- a/excuter/op-mem-cuda/src/deepx/tensorfunc/changeshape_miaobyte.hpp +++ b/excuter/op-mem-cuda/src/deepx/tensorfunc/changeshape_miaobyte.hpp @@ -26,13 +26,16 @@ namespace deepx::tensorfunc { throw std::invalid_argument("Shape size mismatch"); } + Shape newshape(shape); if (tensor.data == output.data) { - output.shape = Shape(shape); + output.shape.shape=newshape.shape; + output.shape.strides=newshape.strides; } else { - output.shape = Shape(shape); + output.shape.shape=newshape.shape; + output.shape.strides=newshape.strides; output.copyer(tensor.data, output.data, tensor.shape.size); } } diff --git a/excuter/op-mem-ompsimd/src/deepx/tensorfunc/changeshape_miaobyte.hpp b/excuter/op-mem-ompsimd/src/deepx/tensorfunc/changeshape_miaobyte.hpp index 2170bc59..ce786bab 100644 --- a/excuter/op-mem-ompsimd/src/deepx/tensorfunc/changeshape_miaobyte.hpp +++ b/excuter/op-mem-ompsimd/src/deepx/tensorfunc/changeshape_miaobyte.hpp @@ -28,13 +28,16 @@ namespace deepx::tensorfunc { throw std::invalid_argument("Shape size mismatch"); } + Shape newshape(shape); if (tensor.data == output.data) { - output.shape = Shape(shape); + output.shape.shape=newshape.shape; + output.shape.strides=newshape.strides; } else { - output.shape = Shape(shape); + output.shape.shape=newshape.shape; + output.shape.strides=newshape.strides; output.copyer(tensor.data, output.data, tensor.shape.size); } } diff --git a/front/py/deepx/nn/deepxir.py b/front/py/deepx/nn/deepxir.py index 988afdc2..4aa08ca5 100644 --- a/front/py/deepx/nn/deepxir.py +++ b/front/py/deepx/nn/deepxir.py @@ -40,8 +40,8 @@ def __init__(self, """ self._name = name - self._args = args - self._returns = returns + self._args = [arg if isinstance(arg, Param) else Param(arg) for arg in args] + self._returns = [ret if isinstance(ret, Param) else Param(ret) for ret in returns] self._author = author self._id=None self._created_at=time.time() diff --git a/front/py/deepx/nn/functional/__init__.py b/front/py/deepx/nn/functional/__init__.py index 4ffa5bc6..c5fc2a8c 100644 --- a/front/py/deepx/nn/functional/__init__.py +++ b/front/py/deepx/nn/functional/__init__.py @@ -4,7 +4,7 @@ from .matmul import matmul from .init import * from .reduce import reduce_max,reduce_min,sum,prod,mean -from .changeshape import transpose,reshape,broadcast_shape,broadcast_to,unsqueeze +from .changeshape import * from .activite import relu,sigmoid,swish __all__ = [ "newtensor", @@ -13,7 +13,7 @@ "add","sub","mul","div","sqrt","pow","exp","log","rsqrt", "matmul", "max","min","sum","prod","mean", - "transpose","reshape","broadcast_shape","broadcast_to","unsqueeze", + "reshape","permute","transpose", "relu","sigmoid","swish", ] \ No newline at end of file diff --git a/front/py/deepx/nn/functional/changeshape.py b/front/py/deepx/nn/functional/changeshape.py index 3209bf3a..294c1dc1 100644 --- a/front/py/deepx/nn/functional/changeshape.py +++ b/front/py/deepx/nn/functional/changeshape.py @@ -2,163 +2,164 @@ from deepx.tensor import Tensor from deepx.nn.deepxir import DeepxIR from deepx.scheduler import send -from deepx.autograd import OpNode - - -#transpose support exchange 2 dimension,or reorder all dimension -OpNode.register("transpose") -def transpose(t: Tensor,dimorder:list[int]=None,inplace:bool=False,out:Union[Tensor,str]='')->Tensor: - if dimorder is None: - return t - - # 处理负数索引并去重 - ndim = t.ndimension - dimorder = [d % ndim for d in dimorder] - unique_dims = list(dict.fromkeys(dimorder)) - - - final_dimorder=[] - if len(unique_dims) == 2: - # 模式1：交换两个维度 - d1, d2 = unique_dims - final_dimorder = list(range(ndim)) - final_dimorder[d1], final_dimorder[d2] = final_dimorder[d2], final_dimorder[d1] - elif len(unique_dims) == ndim: - # 模式2：全排列 - final_dimorder = unique_dims - else: - raise ValueError(f"维度参数不合法，支持两种模式：1.交换两个维度 2.全排列维度顺序，当前输入维度数：{len(unique_dims)}，张量维度数：{ndim}") - +from deepx.autograd import OpNode,Function,Context + +def _A_v_elementwiseop_C( + a:Tensor, + b: list[int] , + op:str=None, + out:Union[Tensor,str]="",author='miaobyte')->Tensor: + g=a.graph + opnode = g.add_op(op) + opnode.add_input(a.node) + opnode.add_input(g.add_vector("",b)) outtensor=None - if inplace: - outtensor=t + if isinstance(out,str): + outtensor=Tensor(shape=b, dtype=a.dtype, device=a.device) + outtensor.addtograph(out) else: - if isinstance(out,str): - outtensor=Tensor(shape=t.Shape.transpose(final_dimorder), dtype=t.dtype, device=t.device) - outtensor.addtograph(out) - else: - outtensor=out - vectornode=t.graph.add_vector("",final_dimorder) - opnode = t.graph.add_op("transpose") - opnode.add_input(t._node) - opnode.add_input(vectornode) - + outtensor=out outtensor.node.add_input(opnode) - if t.graph.eager: - ir=DeepxIR("transpose",'',[t._node.name,*map(str, final_dimorder)], [outtensor._node.name]) + if g.eager: + ir=DeepxIR(op, [a.node.name,b], [outtensor.node.name],author) send(ir) - return outtensor - OpNode.register("reshape") -def reshape(t:Tensor,shape:list[int],inplace:bool=False,out:Union[Tensor,str]='')->Tensor: - outtensor=None - if inplace: - outtensor=t - from deepx import Shape - outtensor._shape=Shape(shape) - else: - if isinstance(out,str): - outtensor=Tensor(shape=shape, dtype=t.dtype, device=t.device) - outtensor.addtograph(out) - else: - outtensor=out - opnode=t.graph.add_op("reshape") - opnode.add_input(t.node) - opnode.add_input(t.graph.add_vector("",shape)) - outtensor.node.add_input(opnode) - if t.graph.eager: - ir=DeepxIR("reshape",'',[t.node.name,*map(str, shape)], [outtensor.node.name]) - send(ir) - return outtensor +class Reshape(Function): + @staticmethod + def forward(ctx:Context,t:Tensor,shape:list[int],out,author='miaobyte'): + ctx.save_data('oldshape',t.shape) + ctx.save_tensors('t',t) + return _A_v_elementwiseop_C(t,shape,"reshape",out,author) + + @staticmethod + def backward(ctx:Context,out_grad): + oldshape=ctx.get_data('oldshape') + t=ctx.get_tensor('t') + return _A_v_elementwiseop_C(out_grad,oldshape,"reshape",t.node.name,author) + +def reshape(t:Tensor,shape:list[int],out:Union[Tensor,str]='')->Tensor: + if t.shape==shape: + return t + return Reshape.apply(t,shape,out) -def broadcast_shape(shape_a: tuple, shape_b: tuple) -> tuple: - """计算两个形状的广播后形状""" - # 获取形状的长度 - len_a, len_b = len(shape_a), len(shape_b) +OpNode.register("transpose") +class Permute(Function): + @staticmethod + def forward(ctx:Context,t:Tensor,dimorder:list[int],out:Union[Tensor,str]='',author='miaobyte')->Tensor: + ctx.save_data('dimorder',dimorder) + ctx.save_tensor('t',t) + return _A_v_elementwiseop_C(t,dimorder,"transpose",out,author) + + @staticmethod + def backward(ctx:Context,in_grad,out_grad,author='miaobyte'): + dimorder=ctx.get_data('dimorder') + inverse_dimorder = [0] * len(dimorder) + for i, j in enumerate(dimorder): + inverse_dimorder[j] = i + return _A_v_elementwiseop_C(out_grad,inverse_dimorder,"transpose",in_grad,author) + +def permute(t:Tensor,dimorder:list[int],out:Union[Tensor,str]='')->Tensor: + if t.dim!=len(dimorder): + raise ValueError(f"shape参数不合法,当前输入维度数：{len(dimorder)}，张量维度数：{t.dim}") + dimorder = [d % t.ndim for d in dimorder] + return Permute.apply(t,dimorder,out) + +def transpose(t: Tensor,out:Union[Tensor,str]='')->Tensor: + dimorder = list(range(t.ndim)) + dimorder[-1],dimorder[-2]=dimorder[-2],dimorder[-1] + return Permute.apply(t,dimorder,out) + + + + +# def broadcast_shape(shape_a: tuple, shape_b: tuple) -> tuple: +# """计算两个形状的广播后形状""" +# # 获取形状的长度 +# len_a, len_b = len(shape_a), len(shape_b) - # 创建结果形状 - result_shape = [] +# # 创建结果形状 +# result_shape = [] - # 从右往左对齐并计算每个维度 - for i in range(1, min(len_a, len_b) + 1): - dim_a = shape_a[-i] - dim_b = shape_b[-i] +# # 从右往左对齐并计算每个维度 +# for i in range(1, min(len_a, len_b) + 1): +# dim_a = shape_a[-i] +# dim_b = shape_b[-i] - if dim_a == 1 or dim_b == 1: - # 广播规则：如果一个维度为1，取另一个维度的值 - result_shape.insert(0, max(dim_a, dim_b)) - elif dim_a == dim_b: - # 维度相同，保持不变 - result_shape.insert(0, dim_a) - else: - # 维度不同且都不为1，无法广播 - raise ValueError(f"无法广播的形状：{shape_a} 和 {shape_b}") +# if dim_a == 1 or dim_b == 1: +# # 广播规则：如果一个维度为1，取另一个维度的值 +# result_shape.insert(0, max(dim_a, dim_b)) +# elif dim_a == dim_b: +# # 维度相同，保持不变 +# result_shape.insert(0, dim_a) +# else: +# # 维度不同且都不为1，无法广播 +# raise ValueError(f"无法广播的形状：{shape_a} 和 {shape_b}") - # 添加较长形状中多出的前导维度 - if len_a > len_b: - result_shape = list(shape_a[:len_a - len_b]) + result_shape - elif len_b > len_a: - result_shape = list(shape_b[:len_b - len_a]) + result_shape +# # 添加较长形状中多出的前导维度 +# if len_a > len_b: +# result_shape = list(shape_a[:len_a - len_b]) + result_shape +# elif len_b > len_a: +# result_shape = list(shape_b[:len_b - len_a]) + result_shape - return tuple(result_shape) +# return tuple(result_shape) -def unsqueeze(t:Tensor,dim:int)->Tensor: - # 确保dim是有效的 - if dim < -t.ndim-1 or dim > t.ndim: - raise ValueError(f"维度超出范围，当前张量维度为{t.ndim},dim={dim}") +# def unsqueeze(t:Tensor,dim:int)->Tensor: +# # 确保dim是有效的 +# if dim < -t.ndim-1 or dim > t.ndim: +# raise ValueError(f"维度超出范围，当前张量维度为{t.ndim},dim={dim}") - # 处理负数索引 - if dim < 0: - dim = t.ndim + dim + 1 - - new_shape = list(t.shape) - new_shape.insert(dim, 1) - - return reshape(t, new_shape) - -OpNode.register("expand") -def expand(t:Tensor,shape:list[int],out:Union[Tensor,str]='')->Tensor: - outtensor=None - if isinstance(out,str): - outtensor=Tensor(shape=shape, dtype=t.dtype, device=t.device) - outtensor.addtograph(out) - else: - outtensor=out - - opnode=t.graph.add_op("expand") - opnode.add_input(t.node) - opnode.add_input(t.graph.add_vector("",shape)) - outtensor.node.add_input(opnode) - if t.graph.eager: - ir=DeepxIR("expand",'',[t.node.name,*map(str, shape)], [outtensor.node.name]) - send(ir) - return outtensor - -def broadcast_to(a: Tensor, shape: tuple,out:Union[Tensor,str]='') -> Tensor: - # 计算广播后的形状 - try: - target_shape = broadcast_shape(a.shape, shape) - if target_shape!=shape: - raise ValueError(f"广播失败：{a.shape} 无法广播为 {shape} ") - except ValueError as e: - raise ValueError(f"广播失败：{e}") from e +# # 处理负数索引 +# if dim < 0: +# dim = t.ndim + dim + 1 + +# new_shape = list(t.shape) +# new_shape.insert(dim, 1) + +# return reshape(t, new_shape) + +# OpNode.register("expand") +# def expand(t:Tensor,shape:list[int],out:Union[Tensor,str]='')->Tensor: +# outtensor=None +# if isinstance(out,str): +# outtensor=Tensor(shape=shape, dtype=t.dtype, device=t.device) +# outtensor.addtograph(out) +# else: +# outtensor=out + +# opnode=t.graph.add_op("expand") +# opnode.add_input(t.node) +# opnode.add_input(t.graph.add_vector("",shape)) +# outtensor.node.add_input(opnode) +# if t.graph.eager: +# ir=DeepxIR("expand",'',[t.node.name,*map(str, shape)], [outtensor.node.name]) +# send(ir) +# return outtensor + +# def broadcast_to(a: Tensor, shape: tuple,out:Union[Tensor,str]='') -> Tensor: +# # 计算广播后的形状 +# try: +# target_shape = broadcast_shape(a.shape, shape) +# if target_shape!=shape: +# raise ValueError(f"广播失败：{a.shape} 无法广播为 {shape} ") +# except ValueError as e: +# raise ValueError(f"广播失败：{e}") from e - # 为每个张量添加前导维度 - if a.shape != target_shape: - a_reshape = [1] * (len(target_shape) - a.ndimension) + list(a.shape) - a_reshaped = reshape(a,a_reshape) - else: - a_reshaped=a +# # 为每个张量添加前导维度 +# if a.shape != target_shape: +# a_reshape = [1] * (len(target_shape) - a.ndimension) + list(a.shape) +# a_reshaped = reshape(a,a_reshape) +# else: +# a_reshaped=a - # 执行实际广播 - if a_reshaped.shape != target_shape: - a_broadcasted = expand(a_reshaped,target_shape,out) - else: - a_broadcasted=a_reshaped +# # 执行实际广播 +# if a_reshaped.shape != target_shape: +# a_broadcasted = expand(a_reshaped,target_shape,out) +# else: +# a_broadcasted=a_reshaped - return a_broadcasted \ No newline at end of file +# return a_broadcasted \ No newline at end of file diff --git a/front/py/deepx/nn/functional/elementwise.py b/front/py/deepx/nn/functional/elementwise.py index 7b524afc..459a21a0 100644 --- a/front/py/deepx/nn/functional/elementwise.py +++ b/front/py/deepx/nn/functional/elementwise.py @@ -3,7 +3,7 @@ from deepx.autograd import Graph,DataNode,OpNode,Function,Context from deepx.nn import DeepxIR,Param from deepx.scheduler import send -from .changeshape import broadcast_shape + def _A_elementwiseop_C( a:Tensor, op:str=None, diff --git a/front/py/deepx/optim/optimizer.py b/front/py/deepx/optim/optimizer.py index 4909b20e..8573025b 100644 --- a/front/py/deepx/optim/optimizer.py +++ b/front/py/deepx/optim/optimizer.py @@ -1,5 +1,9 @@ +from typing import Any +from deepx.tensor import Tensor class Optimizer: - def __init__(self, params,defaults: dict[str, Any]) -> None: + def __init__(self, + params:list[Tensor], + defaults: dict[str, Any]) -> None: self.params = params self.defaults = defaults diff --git a/front/py/deepx/optim/sgd.py b/front/py/deepx/optim/sgd.py index e69de29b..810044b6 100644 --- a/front/py/deepx/optim/sgd.py +++ b/front/py/deepx/optim/sgd.py @@ -0,0 +1,13 @@ +from typing import Any +from .optimizer import Optimizer +from deepx.tensor import Tensor + +class SGD(Optimizer): + def __init__(self, + params:list[Tensor], + defaults: dict[str, Any]) -> None: + super().__init__(params, defaults) + + def step(self): + for param in self.params: + param.data -= self.defaults['lr'] * param.grad diff --git a/front/py/deepx/tensor/changeshape.py b/front/py/deepx/tensor/changeshape.py index b8777b56..b20d6bb4 100644 --- a/front/py/deepx/tensor/changeshape.py +++ b/front/py/deepx/tensor/changeshape.py @@ -1,6 +1,18 @@ from typing import Union from .tensor import Tensor,tensor_method +@tensor_method +def reshape(self,*shape,out:Union[Tensor,str]='')->Tensor: + from deepx.nn.functional import reshape as reshape_func + result=reshape_func(self,shape,out) + return result + +@tensor_method +def reshape_(self,*shape)->Tensor: + from deepx.nn.functional import reshape as reshape_func + result=reshape_func(self,shape,self) + return result + @tensor_method def transpose(self,*axes,out:Union[Tensor,str]=''): from deepx.nn.functional import transpose as transpose_func @@ -13,33 +25,22 @@ def transpose_(self,*axes): transpose_func(self,axes,self) return self -@tensor_method -def reshape(self,*shape,out:Union[Tensor,str]='')->Tensor: - from deepx.nn.functional import reshape as reshape_func - result=reshape_func(self,shape,False,out) - return result -@tensor_method -def reshape_(self,*shape)->Tensor: - from deepx.nn.functional import reshape as reshape_func - result=reshape_func(self,shape,True) - return result - -@tensor_method -def expand(self,shape:tuple)->Tensor: - from deepx.nn.functional import expand as expand_func - result=expand_func(self,shape,False) - return result +# @tensor_method +# def expand(self,shape:tuple)->Tensor: +# from deepx.nn.functional import expand as expand_func +# result=expand_func(self,shape,False) +# return result -@tensor_method -def broadcastshape(self,other:Tensor)->tuple[int]: - from deepx.nn.functional import broadcastshape as broadcastshape_func - result=broadcastshape_func(self.shape,other.shape) - return result +# @tensor_method +# def broadcastshape(self,other:Tensor)->tuple[int]: +# from deepx.nn.functional import broadcastshape as broadcastshape_func +# result=broadcastshape_func(self.shape,other.shape) +# return result -@tensor_method -def broadcast_to(self,shape:tuple,out:Union[Tensor,str]='')->Tensor: - from deepx.nn.functional import broadcast_to as broadcast_to_func - result=broadcast_to_func(self,shape,out) - return result +# @tensor_method +# def broadcast_to(self,shape:tuple,out:Union[Tensor,str]='')->Tensor: +# from deepx.nn.functional import broadcast_to as broadcast_to_func +# result=broadcast_to_func(self,shape,out) +# return result diff --git a/front/py/examples/2_ir/4_changeshape_reshape.dot b/front/py/examples/2_ir/4_changeshape_reshape.dot new file mode 100644 index 00000000..8db2ca57 --- /dev/null +++ b/front/py/examples/2_ir/4_changeshape_reshape.dot @@ -0,0 +1,33 @@ +// Computational Graph +digraph { + rankdir=TB + node [shape=record] + 138483665701232 [label="t1 +(3, 4)" color=skyblue fillcolor=aliceblue fontname="Sans-Serif" labeljust=l shape=box style=filled] + 138481406184944 [label=constant color=darkslategray fillcolor=lightgray fontname="Courier Bold" labeljust=l shape=box style=filled] + 138481407714656 [label="var_1 +1" color=orange fillcolor=moccasin fontname="Sans-Serif" labeljust=l shape=box style=filled] + 138481406184992 [label=reshape color=darkslategray fillcolor=lightgray fontname="Courier Bold" labeljust=l shape=box style=filled] + 138481407705776 [label="vector_1 +(3, 2, 2)" color=darkseagreen fillcolor=honeydew fontname="Sans-Serif" labeljust=l shape=box style=filled] + 138481406185280 [label="tensor_2 +(3, 2, 2)" color=skyblue fillcolor=aliceblue fontname="Sans-Serif" labeljust=l shape=box style=filled] + 138481406185328 [label="tensor_3 +(4, 5)" color=skyblue fillcolor=aliceblue fontname="Sans-Serif" labeljust=l shape=box style=filled] + 138481406185136 [label=constant color=darkslategray fillcolor=lightgray fontname="Courier Bold" labeljust=l shape=box style=filled] + 138481406185712 [label="var_2 +1" color=orange fillcolor=moccasin fontname="Sans-Serif" labeljust=l shape=box style=filled] + 138481406185472 [label=reshape color=darkslategray fillcolor=lightgray fontname="Courier Bold" labeljust=l shape=box style=filled] + 138481406185184 [label="vector_2 +(20,)" color=darkseagreen fillcolor=honeydew fontname="Sans-Serif" labeljust=l shape=box style=filled] + 138481406184944 -> 138483665701232 [arrowsize=0.8 color=gray40 penwidth=1.2] + 138481407714656 -> 138481406184944 [arrowsize=0.8 color=gray40 penwidth=1.2] + 138483665701232 -> 138481406184992 [arrowsize=0.8 color=gray40 penwidth=1.2] + 138481407705776 -> 138481406184992 [arrowsize=0.8 color=gray40 penwidth=1.2] + 138481406184992 -> 138481406185280 [arrowsize=0.8 color=gray40 penwidth=1.2] + 138481406185136 -> 138481406185328 [arrowsize=0.8 color=gray40 penwidth=1.2] + 138481406185472 -> 138481406185328 [arrowsize=0.8 color=gray40 penwidth=1.2] + 138481406185712 -> 138481406185136 [arrowsize=0.8 color=gray40 penwidth=1.2] + 138481406185328 -> 138481406185472 [arrowsize=0.8 color=gray40 penwidth=1.2] + 138481406185184 -> 138481406185472 [arrowsize=0.8 color=gray40 penwidth=1.2] +} diff --git a/front/py/examples/2_ir/4_changeshape_reshape.dot.svg b/front/py/examples/2_ir/4_changeshape_reshape.dot.svg new file mode 100644 index 00000000..4592b180 --- /dev/null +++ b/front/py/examples/2_ir/4_changeshape_reshape.dot.svg @@ -0,0 +1,146 @@ + + + + + + +%3 + + + +138483665701232 + +t1 +(3, 4) + + + +138481406184992 + +reshape + + + +138483665701232->138481406184992 + + + + + +138481406184944 + +constant + + + +138481406184944->138483665701232 + + + + + +138481407714656 + +var_1 +1 + + + +138481407714656->138481406184944 + + + + + +138481406185280 + +tensor_2 +(3, 2, 2) + + + +138481406184992->138481406185280 + + + + + +138481407705776 + +vector_1 +(3, 2, 2) + + + +138481407705776->138481406184992 + + + + + +138481406185328 + +tensor_3 +(4, 5) + + + +138481406185472 + +reshape + + + +138481406185328->138481406185472 + + + + + +138481406185136 + +constant + + + +138481406185136->138481406185328 + + + + + +138481406185712 + +var_2 +1 + + + +138481406185712->138481406185136 + + + + + +138481406185472->138481406185328 + + + + + +138481406185184 + +vector_2 +(20,) + + + +138481406185184->138481406185472 + + + + +