From 5f293645b0c1e45d23f7fcfc4ff41a326178a1f0 Mon Sep 17 00:00:00 2001 From: lipeng <734991033@qq.com> Date: Mon, 23 Jun 2025 22:25:57 +0800 Subject: [PATCH 1/2] =?UTF-8?q?py:=201.tensor=20=5F=5Fgetitem=5F=5F=202.br?= =?UTF-8?q?oadcastTo=EF=BC=8C=E5=A2=9E=E5=8A=A0expand=E3=80=82=E4=B8=A5?= =?UTF-8?q?=E6=A0=BC=E7=BA=A6=E6=9D=9Fbroadcast=5Fshape=203.=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3unsqueeze=204.=E6=96=B0=E5=A2=9Efloat()=E7=AD=89?= =?UTF-8?q?=E5=BF=AB=E6=8D=B7=E7=B1=BB=E5=9E=8B=E8=BD=AC=E6=8D=A2=E6=96=B9?= =?UTF-8?q?=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- front/py/deepx/nn/functional/changeshape.py | 9 +++-- .../nn/functional/leaffunc_changeshape.py | 25 ++++++++---- front/py/deepx/tensor/changeshape.py | 13 ++++--- front/py/deepx/tensor/elementwise.py | 38 ++++++++++++++++++- front/py/deepx/tensor/shape.py | 31 ++++----------- front/py/deepx/tensor/tensor.py | 2 - .../transformer/models/llama/embedding.py | 10 ++--- .../py/examples/1_tensor/getitem/getitem_2.py | 6 +++ 8 files changed, 87 insertions(+), 47 deletions(-) create mode 100644 front/py/examples/1_tensor/getitem/getitem_2.py diff --git a/front/py/deepx/nn/functional/changeshape.py b/front/py/deepx/nn/functional/changeshape.py index 080c6ec..5616c45 100644 --- a/front/py/deepx/nn/functional/changeshape.py +++ b/front/py/deepx/nn/functional/changeshape.py @@ -1,6 +1,6 @@ from typing import Union from deepx import Tensor -from .leaffunc_changeshape import reshape,indexselect, concat +from .leaffunc_changeshape import reshape,indexselect, concat,broadcastTo from .leaffunc_init import newtensor,arange def squeeze(t:Tensor,dim:int)->Tensor: assert isinstance(dim,int) @@ -13,7 +13,7 @@ def squeeze(t:Tensor,dim:int)->Tensor: def unsqueeze(t:Tensor,dim:int)->Tensor: assert isinstance(dim,int) assert isinstance(t,Tensor) - dim=dim%t.ndim + dim = dim % (t.ndim + 1) newshape=list(t.shape) newshape.insert(dim,1) return reshape(t,tuple(newshape)) @@ -29,4 +29,7 @@ def sliceselect(t:Tensor,sliceobj:slice,dim:int=-1,out:Union[Tensor,str]='')->Te index=arange(start,stop,dtype='int32') return indexselect(t,index,dim=dim,out=out) -cat= concat \ No newline at end of file +cat= concat +# 参考 PyTorch 文档，broadcastTo和expand是作用一样 +# https://docs.pytorch.org/docs/stable/generated/torch.broadcast_to.html +expand = broadcastTo \ No newline at end of file diff --git a/front/py/deepx/nn/functional/leaffunc_changeshape.py b/front/py/deepx/nn/functional/leaffunc_changeshape.py index 5f60bda..750378c 100644 --- a/front/py/deepx/nn/functional/leaffunc_changeshape.py +++ b/front/py/deepx/nn/functional/leaffunc_changeshape.py @@ -61,16 +61,25 @@ def concat(tensors:Union[list[Tensor],tuple[Tensor,...]],dim:int,out:Union[Tenso rtf_concat(tensors,dim,outtensor,defaultauthor['concat']) return outtensor -def broadcastTo(t:Tensor,new_shape:tuple[int,...],out:Union[Tensor,str]='',requires_grad:bool=False,author='miaobyte')->Tensor: - assert isinstance(new_shape,tuple) - for i in new_shape: - assert isinstance(i,int) and i>0 - +def broadcastTo(t:Tensor,newshape:tuple[int,...],out:Union[Tensor,str]='')->Tensor: + assert isinstance(newshape,tuple) + assert len(newshape)==t.ndim + new_shape=[] + for idx,i in enumerate(newshape): + assert isinstance(i,int) + if i==-1: + new_shape.append(t.shape[idx]) + elif i>0: + new_shape.append(i) + else: + raise ValueError(f"新形状参数不合法，维度 {idx} 的值{i}") + new_shape=tuple(new_shape) if t.shape==new_shape: return t + bshape=Shape.broadcast_shape(t.shape,new_shape) - if bshape!=tuple(new_shape): - raise ValueError(f"广播失败：{t.shape} 无法广播为 {new_shape} ") + if bshape!=new_shape: + raise ValueError(f"广播失败：{t.shape} 无法广播为 {new_shape}，请先reshape") outtensor=out if isinstance(out,str) or out is None: outshape=new_shape @@ -78,8 +87,10 @@ def broadcastTo(t:Tensor,new_shape:tuple[int,...],out:Union[Tensor,str]='',requi from .rtf_changeshape import rtf_broadcastTo rtf_broadcastTo(t,new_shape,outtensor,defaultauthor['broadcastTo']) return outtensor + broadcast_to = broadcastTo + def indexselect(input:Tensor,indices:Tensor,dim:int,out:Union[Tensor,str]='')->Tensor: assert dim>=0 and dimTensor: from deepx.nn.functional import broadcastTo as broadcastTo_func @@ -54,6 +56,12 @@ def broadcast_to(self,shape:tuple[int,...],out:Union[Tensor,str]='')->Tensor: result=broadcast_to_func(self,shape,out) return result +@tensor_method +def expand(self,shape:tuple[int,...],out:Union[Tensor,str]='')->Tensor: + from deepx.nn.functional import broadcastTo as expand_func + result=expand_func(self,shape,out) + return result + @tensor_method def indexselect(self,index:Tensor,gatheraxis:int=0,out:Union[Tensor,str]='')->Tensor: assert isinstance(index,Tensor) @@ -88,8 +96,3 @@ def repeat(self,repeats:tuple[int,...])->Tensor: result=repeat_func(self,repeats) return result -# @tensor_method -# def expand(self,shape:tuple)->Tensor: -# from deepx.nn.functional import expand as expand_func -# result=expand_func(self,shape,False) -# return result diff --git a/front/py/deepx/tensor/elementwise.py b/front/py/deepx/tensor/elementwise.py index 3aafd7a..b5e0d93 100644 --- a/front/py/deepx/tensor/elementwise.py +++ b/front/py/deepx/tensor/elementwise.py @@ -228,9 +228,45 @@ def switch_(self,cases:Union[Tensor,float,int]): # 类型转换 @tensor_method def todtype(self,dtype:str): + assert isinstance(dtype,str) + if str== self.dtype: + return self from deepx.nn.functional import newtensor dest=newtensor(self.shape,dtype=dtype) from deepx.nn.functional import todtype as todtype_func todtype_func(self,dest) return dest - \ No newline at end of file + +@tensor_method +def double(self)->Tensor: + """将张量转换为float64类型""" + return self.todtype('float64') + +@tensor_method +def float(self)->Tensor: + """将张量转换为float32类型""" + return self.todtype('float32') + +@tensor_method +def half(self)->Tensor: + """将张量转换为float16类型""" + return self.todtype('float16') + +@tensor_method +def long(self)->Tensor: + """将张量转换为int64类型""" + return self.todtype('int64') + + +@tensor_method +def int(self)->Tensor: + """将张量转换为int32类型""" + return self.todtype('int32') + +@tensor_method +def bool(self)->Tensor: + """将张量转换为bool类型""" + return self.todtype('bool') + + + diff --git a/front/py/deepx/tensor/shape.py b/front/py/deepx/tensor/shape.py index 1cd165c..f7e9262 100644 --- a/front/py/deepx/tensor/shape.py +++ b/front/py/deepx/tensor/shape.py @@ -142,34 +142,17 @@ def matmul(cls,shape:tuple[int],other:tuple[int])->tuple[int]: @classmethod def broadcast_shape(cls,shape_a: tuple[int,...], shape_b: tuple[int,...]) -> tuple[int,...]: - """计算两个形状的广播后形状""" - # 获取形状的长度 - len_a, len_b = len(shape_a), len(shape_b) - - # 创建结果形状 + assert isinstance(shape_a,tuple) and isinstance(shape_b,tuple) + assert len(shape_b)==len(shape_a) + """计算两个形状的广播后形状（长度必须一致）""" result_shape = [] - - # 从右往左对齐并计算每个维度 - for i in range(1, min(len_a, len_b) + 1): - dim_a = shape_a[-i] - dim_b = shape_b[-i] - + for dim_a, dim_b in zip(shape_a, shape_b): if dim_a == 1 or dim_b == 1: - # 广播规则：如果一个维度为1，取另一个维度的值 - result_shape.insert(0, max(dim_a, dim_b)) + result_shape.append(max(dim_a, dim_b)) elif dim_a == dim_b: - # 维度相同，保持不变 - result_shape.insert(0, dim_a) + result_shape.append(dim_a) else: - # 维度不同且都不为1，无法广播 - raise ValueError(f"无法广播的形状：{shape_a} 和 {shape_b}") - - # 添加较长形状中多出的前导维度 - if len_a > len_b: - result_shape = list(shape_a[:len_a - len_b]) + result_shape - elif len_b > len_a: - result_shape = list(shape_b[:len_b - len_a]) + result_shape - + raise ValueError(f"无法广播的形状：{shape_a} 和 {shape_b},请先reshape") return tuple(result_shape) diff --git a/front/py/deepx/tensor/tensor.py b/front/py/deepx/tensor/tensor.py index 1566e21..5a1244b 100644 --- a/front/py/deepx/tensor/tensor.py +++ b/front/py/deepx/tensor/tensor.py @@ -1,7 +1,5 @@ from typing import Optional,Union,TypeAlias - from triton.language.semantic import equal - from .shape import Shape diff --git a/front/py/deepx/transformer/models/llama/embedding.py b/front/py/deepx/transformer/models/llama/embedding.py index fbccc0a..2de28f5 100644 --- a/front/py/deepx/transformer/models/llama/embedding.py +++ b/front/py/deepx/transformer/models/llama/embedding.py @@ -39,12 +39,12 @@ def __init__(self,config:dict): def forward(self, x, position_ids): # 扩展旋转频率 - inv_freq_expanded = self.inv_freq.unsqueeze(dim=0).unsqueeze(dim=2).todtype('float32') - broadcast_shape=(position_ids.shape[0], self.inv_freq.shape[0], 1) - inv_freq_expanded = inv_freq_expanded.reshape(broadcast_shape) - + inv_freq_expanded = self.inv_freq[None, :, None].todtype('float32').expand((position_ids.shape[0], -1, 1)) + # 使用torch.unsqueeze和type转换替代索引操作 - position_ids_expanded = position_ids.unsqueeze(dim=1).todtype(x.dtype) + position_ids_expanded = position_ids[:, None, :].float() + + # 计算频率 freqs = (inv_freq_expanded @ position_ids_expanded).T # 拼接频率 diff --git a/front/py/examples/1_tensor/getitem/getitem_2.py b/front/py/examples/1_tensor/getitem/getitem_2.py new file mode 100644 index 0000000..393356c --- /dev/null +++ b/front/py/examples/1_tensor/getitem/getitem_2.py @@ -0,0 +1,6 @@ +from deepx import newtensor,arange +t = newtensor((64,)) +t.arange_() +print() +t2 = t[None, :, None] +t2.print() \ No newline at end of file From 3c361a1acfc3e674102d6855b13de1c2ea584051 Mon Sep 17 00:00:00 2001 From: lipeng <734991033@qq.com> Date: Mon, 23 Jun 2025 22:54:37 +0800 Subject: [PATCH 2/2] =?UTF-8?q?py:=201.=E9=87=8D=E6=96=B0=E8=BF=9B?= =?UTF-8?q?=E8=A1=8C=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- front/py/examples/2_ir/4_changeshape_broadcast.py | 4 ++-- .../examples/2_ir/4_changeshape_broadcast_add.py | 15 --------------- front/py/examples/2_ir/4_changeshape_concat.py | 8 ++++---- .../3_functional/changeshape_broadcast.py | 2 +- front/py/examples/3_module/1_linear.py | 2 +- 5 files changed, 8 insertions(+), 23 deletions(-) delete mode 100644 front/py/examples/2_ir/4_changeshape_broadcast_add.py diff --git a/front/py/examples/2_ir/4_changeshape_broadcast.py b/front/py/examples/2_ir/4_changeshape_broadcast.py index 3c8f77c..fd1d126 100644 --- a/front/py/examples/2_ir/4_changeshape_broadcast.py +++ b/front/py/examples/2_ir/4_changeshape_broadcast.py @@ -11,11 +11,11 @@ ########====DEEPX====######## -from deepx import Tensor,arange,broadcastTo +from deepx import arange a=arange(start=0,end=4*2*3,name="a").reshape_((4,2,3)) b=arange(start=0,end=2,name='b').reshape((2,1)) -bb=b.broadcastTo( a.shape,out="b.broadcasted") +bb=b.unsqueeze(0).broadcastTo(a.shape,out="b.broadcasted") bb.print() c=a[None:,] diff --git a/front/py/examples/2_ir/4_changeshape_broadcast_add.py b/front/py/examples/2_ir/4_changeshape_broadcast_add.py deleted file mode 100644 index 7ebd743..0000000 --- a/front/py/examples/2_ir/4_changeshape_broadcast_add.py +++ /dev/null @@ -1,15 +0,0 @@ -########====DEEPX====######## -from deepx import ones - -a=ones( 4,2,3 ,name="a") -b=ones( 2,1 ,name='b') -c=a+b -print(c) - -########====pytorch====######## -import torch -torch_a=torch.ones(4,2,3) -torch_b=torch.ones(2,1) -torch_c=torch_a+torch_b -print(torch_c) - diff --git a/front/py/examples/2_ir/4_changeshape_concat.py b/front/py/examples/2_ir/4_changeshape_concat.py index ada8e0b..a8dc9bd 100644 --- a/front/py/examples/2_ir/4_changeshape_concat.py +++ b/front/py/examples/2_ir/4_changeshape_concat.py @@ -15,9 +15,9 @@ from deepx import Tensor,zeros, ones, concat -t1 = ones([3,4,5],dtype='float32',name='t1') -t2=ones([3,4,5],dtype='float32',name='t2') -t3=ones([3,4,5],dtype='float32',name='t3') +t1 = ones( (3,4,5),dtype='float32',name='t1') +t2=ones((3,4,5),dtype='float32',name='t2') +t3=ones((3,4,5),dtype='float32',name='t3') -t=concat([t1,t2,t3],dim=1,out='t') +t=concat((t1,t2,t3),dim=1,out='t') t.print() diff --git a/front/py/examples/3_functional/changeshape_broadcast.py b/front/py/examples/3_functional/changeshape_broadcast.py index 8721041..d1b9b92 100644 --- a/front/py/examples/3_functional/changeshape_broadcast.py +++ b/front/py/examples/3_functional/changeshape_broadcast.py @@ -13,7 +13,7 @@ #######-----------------deepx-----------------####### from deepx import Tensor,broadcast_to,arange deepx_x = arange(0,6).reshape_((1,2,3)) # shape=(2,3) -deepx_y = broadcast_to(deepx_x, (3,2,3)) # 需要原维度为1 +deepx_y = deepx_x.broadcast_to((3,2,3)) # 需要原维度为1 deepx_y.print() diff --git a/front/py/examples/3_module/1_linear.py b/front/py/examples/3_module/1_linear.py index 7ad43a9..47de02e 100644 --- a/front/py/examples/3_module/1_linear.py +++ b/front/py/examples/3_module/1_linear.py @@ -14,7 +14,7 @@ from deepx import ones net = Linear(64, 4) -input=ones(1,64,name='input') +input=ones((1,64),name='input') out=net.forward(input) out.print()