From 8027e1530fd3da8cd7796e1c1fd7890450ef48e1 Mon Sep 17 00:00:00 2001 From: lipeng <734991033@qq.com> Date: Wed, 23 Apr 2025 21:20:22 +0800 Subject: [PATCH 1/4] =?UTF-8?q?deepxctl:=20=20=20deepxctl,golang=E5=AE=9E?= =?UTF-8?q?=E7=8E=B0=EF=BC=8C=E6=8F=90=E4=BE=9B=E7=BB=9F=E4=B8=80=E5=91=BD?= =?UTF-8?q?=E4=BB=A4=E8=A1=8C=E8=BF=90=E7=BB=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/tool-deepxctl.yml | 37 +++++++++ deepxctl/.gitignore | 2 - deepxctl/cmd/tensor/print.go | 83 ------------------- deepxctl/cmd/tensor/tensor.go | 36 --------- deepxctl/go.mod | 5 -- deepxctl/go.sum | 3 - deepxctl/main.go | 64 --------------- deepxctl/tensor/fp16.go | 28 ------- deepxctl/tensor/io.go | 47 ----------- deepxctl/tensor/print.go | 121 ---------------------------- deepxctl/tensor/tensor.go | 90 --------------------- 11 files changed, 37 insertions(+), 479 deletions(-) create mode 100644 .github/workflows/tool-deepxctl.yml delete mode 100644 deepxctl/.gitignore delete mode 100644 deepxctl/cmd/tensor/print.go delete mode 100644 deepxctl/cmd/tensor/tensor.go delete mode 100644 deepxctl/go.mod delete mode 100644 deepxctl/go.sum delete mode 100644 deepxctl/main.go delete mode 100644 deepxctl/tensor/fp16.go delete mode 100644 deepxctl/tensor/io.go delete mode 100644 deepxctl/tensor/print.go delete mode 100644 deepxctl/tensor/tensor.go diff --git a/.github/workflows/tool-deepxctl.yml b/.github/workflows/tool-deepxctl.yml new file mode 100644 index 00000000..a3c13ad5 --- /dev/null +++ b/.github/workflows/tool-deepxctl.yml @@ -0,0 +1,37 @@ +name: Tool/deepxctl Build +on: [push, pull_request] + +jobs: + build: + strategy: + matrix: + os: [ubuntu-22.04] # 只保留 Ubuntu + go-version: [1.23.2] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # 系统依赖安装 + - name: 安装Go + uses: actions/setup-go@v4 + with: + go-version: ${{ matrix.go-version }} + cache: true + + # 系统依赖安装 + - name: 安装依赖 (Ubuntu) + env: + DEBIAN_FRONTEND: noninteractive + run: | + sudo apt-get update + sudo apt-get install -y git + + # 构建deepxctl工具 + - name: 构建deepxctl + run: | + cd tool/deepxctl + go build -v -o deepxctl + \ No newline at end of file diff --git a/deepxctl/.gitignore b/deepxctl/.gitignore deleted file mode 100644 index 5ca0477f..00000000 --- a/deepxctl/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -.idea -deepxctl \ No newline at end of file diff --git a/deepxctl/cmd/tensor/print.go b/deepxctl/cmd/tensor/print.go deleted file mode 100644 index 44029497..00000000 --- a/deepxctl/cmd/tensor/print.go +++ /dev/null @@ -1,83 +0,0 @@ -package tensor - -import ( - "flag" - "fmt" - "os" - - coretensor "github.com/array2d/deepx/deepxctl/tensor" -) - -func PrintCmd() { - printCmd := flag.NewFlagSet("print", flag.ExitOnError) - tensorPath := os.Args[0] - if tensorPath == "" { - fmt.Println("请指定文件路径") - printCmd.Usage() - return - } - var err error - var shape coretensor.Shape - shape, err = coretensor.LoadShape(tensorPath) - if err != nil { - fmt.Println("读取文件失败:", err) - } - switch shape.Dtype { - case "bool": - var t coretensor.Tensor[bool] - t, err = coretensor.LoadTensor[bool](tensorPath) - if err != nil { - fmt.Println("读取文件失败:", err) - } - t.Print() - case "int8": - var t coretensor.Tensor[int8] - t, err = coretensor.LoadTensor[int8](tensorPath) - if err != nil { - fmt.Println("读取文件失败:", err) - } - t.Print() - case "int16": - var t coretensor.Tensor[int16] - t, err = coretensor.LoadTensor[int16](tensorPath) - if err != nil { - fmt.Println("读取文件失败:", err) - } - t.Print() - case "int32": - var t coretensor.Tensor[int32] - t, err = coretensor.LoadTensor[int32](tensorPath) - if err != nil { - fmt.Println("读取文件失败:", err) - } - t.Print() - case "int64": - var t coretensor.Tensor[int64] - t, err = coretensor.LoadTensor[int64](tensorPath) - if err != nil { - fmt.Println("读取文件失败:", err) - } - t.Print() - case "float16": - // var t coretensor.Tensor[float16] - // t, err = coretensor.LoadTensor[float16](tensorPath) - // if err != nil { - // fmt.Println("读取文件失败:", err) - // } - // t.Print() - case "float32": - var t coretensor.Tensor[float32] - t, err = coretensor.LoadTensor[float32](tensorPath) - if err != nil { - fmt.Println("读取文件失败:", err) - } - t.Print() - case "float64": - var t coretensor.Tensor[float64] - t, err = coretensor.LoadTensor[float64](tensorPath) - if err != nil { - fmt.Println("读取文件失败:", err) - } - t.Print() - } -} diff --git a/deepxctl/cmd/tensor/tensor.go b/deepxctl/cmd/tensor/tensor.go deleted file mode 100644 index eb11dba0..00000000 --- a/deepxctl/cmd/tensor/tensor.go +++ /dev/null @@ -1,36 +0,0 @@ -package tensor - -import ( - "fmt" - "os" -) - -func PrintUsage() { - fmt.Println("使用方法:") - fmt.Println(" tensor print <文件路径>") - fmt.Println(" tensor help") -} - -func Execute() { - if len(os.Args) < 1 { - PrintUsage() - os.Exit(1) - } - - subCmd := "help" - if len(os.Args) > 0 { - subCmd = os.Args[0] - } - - switch subCmd { - case "print": - os.Args = os.Args[1:] - PrintCmd() - case "help": - PrintUsage() - default: - fmt.Printf("未知的张量命令: %s\n", subCmd) - PrintUsage() - os.Exit(1) - } -} diff --git a/deepxctl/go.mod b/deepxctl/go.mod deleted file mode 100644 index 8cabf45f..00000000 --- a/deepxctl/go.mod +++ /dev/null @@ -1,5 +0,0 @@ -module github.com/array2d/deepx/deepxctl - -go 1.23.2 - -require gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/deepxctl/go.sum b/deepxctl/go.sum deleted file mode 100644 index 75346616..00000000 --- a/deepxctl/go.sum +++ /dev/null @@ -1,3 +0,0 @@ -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/deepxctl/main.go b/deepxctl/main.go deleted file mode 100644 index de73e43c..00000000 --- a/deepxctl/main.go +++ /dev/null @@ -1,64 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "os" - "path/filepath" - - "github.com/array2d/deepx/deepxctl/cmd/tensor" -) - -var version = "0.1.0" - -func printUsage() { - execName := filepath.Base(os.Args[0]) - fmt.Printf("用法: %s [命令] [参数]\n\n", execName) - fmt.Println("可用命令:") - fmt.Println(" tensor 张量操作相关命令") - fmt.Println(" version 显示版本信息") - fmt.Println(" help 显示帮助信息") - fmt.Println("\n使用 '%s help [命令]' 获取命令的详细信息", execName) -} - -func main() { - flag.Usage = printUsage - - if len(os.Args) < 2 { - printUsage() - os.Exit(1) - } - - // 获取子命令 - cmd := os.Args[1] - - // 根据子命令执行相应操作 - switch cmd { - case "tensor": - // 移除子命令，让子命令处理剩余的参数 - os.Args = os.Args[2:] - tensor.Execute() - - case "version": - fmt.Printf("deepxctl 版本 %s\n", version) - - case "help": - if len(os.Args) > 2 { - helpCmd := os.Args[2] - switch helpCmd { - case "tensor": - tensor.PrintUsage() - default: - fmt.Printf("未知命令: %s\n", helpCmd) - printUsage() - } - } else { - printUsage() - } - - default: - fmt.Printf("未知命令: %s\n", cmd) - printUsage() - os.Exit(1) - } -} diff --git a/deepxctl/tensor/fp16.go b/deepxctl/tensor/fp16.go deleted file mode 100644 index 18a7e0f8..00000000 --- a/deepxctl/tensor/fp16.go +++ /dev/null @@ -1,28 +0,0 @@ -package tensor - -import ( - "encoding/binary" - "math" -) - -func Byte2ToFloat16(value []byte) float32 { - bits := binary.BigEndian.Uint16(value) - // 这里需要实现float16到float32的转换 - // 简化实现，实际项目中需要更完整的实现 - sign := float32(1) - if bits&0x8000 != 0 { - sign = -1 - } - exp := int((bits & 0x7C00) >> 10) - frac := float32(bits&0x03FF) / 1024.0 - - if exp == 0 { - return sign * frac * float32(1.0/16384.0) // 非规格化数 - } else if exp == 31 { - if frac == 0 { - return sign * float32(math.Inf(1)) // 无穷大 - } - return float32(math.NaN()) // NaN - } - return sign * float32(math.Pow(2, float64(exp-15))) * (1.0 + frac) // 规格化数 -} diff --git a/deepxctl/tensor/io.go b/deepxctl/tensor/io.go deleted file mode 100644 index b0faf9d7..00000000 --- a/deepxctl/tensor/io.go +++ /dev/null @@ -1,47 +0,0 @@ -package tensor - -import ( - "encoding/binary" - "os" - - "gopkg.in/yaml.v2" -) - -func LoadShape(filePath string) (shape Shape, err error) { - var shapeData []byte - shapeData, err = os.ReadFile(filePath + ".shape") - if err != nil { - return - } - - err = yaml.Unmarshal(shapeData, &shape) - if err != nil { - return - } - return -} -func LoadTensor[T Number](filePath string) (tensor Tensor[T], err error) { - - _, err = os.ReadFile(filePath + ".shape") - if err != nil { - return - } - var shape Shape - shape, err = LoadShape(filePath) - if err != nil { - return - } - file, err := os.Open(filePath + ".data") - if err != nil { - return - } - defer file.Close() - data := make([]T, shape.Size) - - err = binary.Read(file, binary.LittleEndian, data) - if err != nil { - return - } - tensor = Tensor[T]{Data: data, Shape: shape} - return -} diff --git a/deepxctl/tensor/print.go b/deepxctl/tensor/print.go deleted file mode 100644 index 89dd3535..00000000 --- a/deepxctl/tensor/print.go +++ /dev/null @@ -1,121 +0,0 @@ -package tensor - -import "fmt" - -func (t *Tensor[T]) Range(dimCount int, f func(indices []int)) { - Shape := t.Shape - if dimCount > len(Shape.Shape) { - panic("dimCount exceeds the number of dimensions in the Tensor.") - } - - totalSize := 1 - - // 计算总的循环次数 - for i := 0; i < dimCount; i++ { - totalSize *= Shape.At(i) - } - indices := make([]int, dimCount) // 初始化索引向量 - // 遍历所有可能的索引组合 - for idx := 0; idx < totalSize; idx++ { - // 反算出 indices 数组 - idx_ := idx - for dim := dimCount - 1; dim >= 0; dim-- { - indices[dim] = idx_ % Shape.At(dim) // 计算当前维度的索引 - idx_ /= Shape.At(dim) // 更新 idx - } - f(indices) // 调用传入的函数 - } -} - -func AutoFormat(dtype string) string { - switch dtype { - case "bool": - return "%v" - case "int8": - return "%d" - case "int16": - return "%d" - case "int32": - return "%d" - case "int64": - return "%d" - case "float16": - return "%f" - case "float32": - return "%f" - case "float64": - return "%f" - default: - return "%v" - } -} - -// Print 打印Tensor的值 -func (t *Tensor[T]) Print(format_ ...string) { - Shape := t.Shape - format := AutoFormat(t.Dtype) - if len(format_) > 0 { - format = format_[0] - } - fmt.Print("shape:[") - for i := 0; i < Shape.Dim; i++ { - fmt.Print(Shape.At(i)) - if i < Shape.Dim-1 { - fmt.Print(", ") - } - } - fmt.Println("]") - if Shape.Dim == 1 { - fmt.Print("[") - for i := 0; i < Shape.At(0); i++ { - if i > 0 { - fmt.Print(" ") - } - fmt.Printf(format, t.Get(i)) - } - fmt.Println("]") - } else if Shape.Dim == 2 { - fmt.Println("[") - for i := 0; i < Shape.At(0); i++ { - fmt.Print(" [") - for j := 0; j < Shape.At(1); j++ { - if j > 0 { - fmt.Print(" ") - } - fmt.Printf(format, t.Get(i, j)) - } - - fmt.Print("]") - if i < Shape.At(0)-1 { - fmt.Print(",") - } - fmt.Println() - } - fmt.Println("]") - } else { - t.Range(Shape.Dim-2, func(indices []int) { - fmt.Print(indices) - m, n := Shape.At(Shape.Dim-2), Shape.At(Shape.Dim-1) - fmt.Print([]int{m, n}) - fmt.Println("=") - - fmt.Println("[") - for i := 0; i < m; i++ { - fmt.Print(" [") - for j := 0; j < n; j++ { - if j > 0 { - fmt.Print(" ") - } - fmt.Printf(format, t.Get(append(indices, i, j)...)) - } - - fmt.Print("]") - if i < m-1 { - fmt.Print(",") - } - fmt.Println() - } - fmt.Println("]") - }) - } -} diff --git a/deepxctl/tensor/tensor.go b/deepxctl/tensor/tensor.go deleted file mode 100644 index 0ef542b3..00000000 --- a/deepxctl/tensor/tensor.go +++ /dev/null @@ -1,90 +0,0 @@ -package tensor - -import ( - "fmt" -) - -type Shape struct { - Shape []int `json:"shape"` - Stride []int `json:"stride"` - Dim int `json:"ndim"` - Size int `json:"size"` - Dtype string `json:"dtype"` -} - -func NewTensorShape(shape []int) (s Shape) { - s.Dim = len(shape) - s.Shape = make([]int, len(shape)) - copy(s.Shape, shape) - s.Stride = make([]int, len(shape)) - s.Stride[len(shape)-1] = 1 - for i := len(shape) - 2; i >= 0; i-- { - s.Stride[i] = s.Stride[i+1] * shape[i+1] - } - s.Size = s.Stride[0] * shape[0] - return s -} -func (s Shape) String() string { - return fmt.Sprintf("%v", s.Shape) -} - -func (s Shape) At(i int) int { - return s.Shape[i] -} - -func (s Shape) LinearAt(indices []int) int { - idx := 0 - for i := 0; i < len(indices); i++ { - idx += indices[i] * s.Stride[i] - } - return idx -} -func (s Shape) LinearTo(idx int) (indices []int) { - linearIndex := idx - indices = make([]int, s.Dim) - for i := 0; i < s.Dim; i++ { - indices[i] = linearIndex / s.Stride[i] - linearIndex %= s.Stride[i] - } - return indices -} - -func BitSize(Dtype string) int { - switch Dtype { - case "bool": - return 8 - case "int8": - return 8 - case "int16": - return 16 - case "int32": - return 32 - case "int64": - return 64 - case "float16": - return 16 - case "float32": - return 32 - case "float64": - return 64 - default: - return 0 - } -} - -type Number interface { - comparable - float64 | float32 | int64 | int32 | int16 | int8 | bool -} - -type Tensor[T Number] struct { - Data []T - Shape -} - -// Get 获取Tensor的值 -func (t *Tensor[T]) Get(indices ...int) T { - idx := t.Shape.LinearAt(indices) - return t.Data[idx] - -} From 1770525d5b71a354405202dd0e3f271e5080bd30 Mon Sep 17 00:00:00 2001 From: lipeng <734991033@qq.com> Date: Wed, 23 Apr 2025 21:41:02 +0800 Subject: [PATCH 2/4] /tool/deepxctl --- tool/deepxctl/.gitignore | 2 + tool/deepxctl/cmd/tensor/print.go | 83 ++++++++++++++++++++ tool/deepxctl/cmd/tensor/tensor.go | 36 +++++++++ tool/deepxctl/go.mod | 5 ++ tool/deepxctl/go.sum | 3 + tool/deepxctl/main.go | 64 +++++++++++++++ tool/deepxctl/tensor/fp16.go | 28 +++++++ tool/deepxctl/tensor/io.go | 47 +++++++++++ tool/deepxctl/tensor/print.go | 121 +++++++++++++++++++++++++++++ tool/deepxctl/tensor/tensor.go | 90 +++++++++++++++++++++ 10 files changed, 479 insertions(+) create mode 100644 tool/deepxctl/.gitignore create mode 100644 tool/deepxctl/cmd/tensor/print.go create mode 100644 tool/deepxctl/cmd/tensor/tensor.go create mode 100644 tool/deepxctl/go.mod create mode 100644 tool/deepxctl/go.sum create mode 100644 tool/deepxctl/main.go create mode 100644 tool/deepxctl/tensor/fp16.go create mode 100644 tool/deepxctl/tensor/io.go create mode 100644 tool/deepxctl/tensor/print.go create mode 100644 tool/deepxctl/tensor/tensor.go diff --git a/tool/deepxctl/.gitignore b/tool/deepxctl/.gitignore new file mode 100644 index 00000000..5ca0477f --- /dev/null +++ b/tool/deepxctl/.gitignore @@ -0,0 +1,2 @@ +.idea +deepxctl \ No newline at end of file diff --git a/tool/deepxctl/cmd/tensor/print.go b/tool/deepxctl/cmd/tensor/print.go new file mode 100644 index 00000000..f755bfcd --- /dev/null +++ b/tool/deepxctl/cmd/tensor/print.go @@ -0,0 +1,83 @@ +package tensor + +import ( + "flag" + "fmt" + "os" + + coretensor "github.com/array2d/deepx/tool/deepxctl/tensor" +) + +func PrintCmd() { + printCmd := flag.NewFlagSet("print", flag.ExitOnError) + tensorPath := os.Args[0] + if tensorPath == "" { + fmt.Println("请指定文件路径") + printCmd.Usage() + return + } + var err error + var shape coretensor.Shape + shape, err = coretensor.LoadShape(tensorPath) + if err != nil { + fmt.Println("读取文件失败:", err) + } + switch shape.Dtype { + case "bool": + var t coretensor.Tensor[bool] + t, err = coretensor.LoadTensor[bool](tensorPath) + if err != nil { + fmt.Println("读取文件失败:", err) + } + t.Print() + case "int8": + var t coretensor.Tensor[int8] + t, err = coretensor.LoadTensor[int8](tensorPath) + if err != nil { + fmt.Println("读取文件失败:", err) + } + t.Print() + case "int16": + var t coretensor.Tensor[int16] + t, err = coretensor.LoadTensor[int16](tensorPath) + if err != nil { + fmt.Println("读取文件失败:", err) + } + t.Print() + case "int32": + var t coretensor.Tensor[int32] + t, err = coretensor.LoadTensor[int32](tensorPath) + if err != nil { + fmt.Println("读取文件失败:", err) + } + t.Print() + case "int64": + var t coretensor.Tensor[int64] + t, err = coretensor.LoadTensor[int64](tensorPath) + if err != nil { + fmt.Println("读取文件失败:", err) + } + t.Print() + case "float16": + // var t coretensor.Tensor[float16] + // t, err = coretensor.LoadTensor[float16](tensorPath) + // if err != nil { + // fmt.Println("读取文件失败:", err) + // } + // t.Print() + case "float32": + var t coretensor.Tensor[float32] + t, err = coretensor.LoadTensor[float32](tensorPath) + if err != nil { + fmt.Println("读取文件失败:", err) + } + t.Print() + case "float64": + var t coretensor.Tensor[float64] + t, err = coretensor.LoadTensor[float64](tensorPath) + if err != nil { + fmt.Println("读取文件失败:", err) + } + t.Print() + } +} diff --git a/tool/deepxctl/cmd/tensor/tensor.go b/tool/deepxctl/cmd/tensor/tensor.go new file mode 100644 index 00000000..eb11dba0 --- /dev/null +++ b/tool/deepxctl/cmd/tensor/tensor.go @@ -0,0 +1,36 @@ +package tensor + +import ( + "fmt" + "os" +) + +func PrintUsage() { + fmt.Println("使用方法:") + fmt.Println(" tensor print <文件路径>") + fmt.Println(" tensor help") +} + +func Execute() { + if len(os.Args) < 1 { + PrintUsage() + os.Exit(1) + } + + subCmd := "help" + if len(os.Args) > 0 { + subCmd = os.Args[0] + } + + switch subCmd { + case "print": + os.Args = os.Args[1:] + PrintCmd() + case "help": + PrintUsage() + default: + fmt.Printf("未知的张量命令: %s\n", subCmd) + PrintUsage() + os.Exit(1) + } +} diff --git a/tool/deepxctl/go.mod b/tool/deepxctl/go.mod new file mode 100644 index 00000000..42c0efe6 --- /dev/null +++ b/tool/deepxctl/go.mod @@ -0,0 +1,5 @@ +module github.com/array2d/deepx/tool/deepxctl + +go 1.23.2 + +require gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/tool/deepxctl/go.sum b/tool/deepxctl/go.sum new file mode 100644 index 00000000..75346616 --- /dev/null +++ b/tool/deepxctl/go.sum @@ -0,0 +1,3 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/tool/deepxctl/main.go b/tool/deepxctl/main.go new file mode 100644 index 00000000..f1f72af2 --- /dev/null +++ b/tool/deepxctl/main.go @@ -0,0 +1,64 @@ +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + + "github.com/array2d/deepx/tool/deepxctl/cmd/tensor" +) + +var version = "0.1.0" + +func printUsage() { + execName := filepath.Base(os.Args[0]) + fmt.Printf("用法: %s [命令] [参数]\n\n", execName) + fmt.Println("可用命令:") + fmt.Println(" tensor 张量操作相关命令") + fmt.Println(" version 显示版本信息") + fmt.Println(" help 显示帮助信息") + fmt.Println("\n使用 '%s help [命令]' 获取命令的详细信息", execName) +} + +func main() { + flag.Usage = printUsage + + if len(os.Args) < 2 { + printUsage() + os.Exit(1) + } + + // 获取子命令 + cmd := os.Args[1] + + // 根据子命令执行相应操作 + switch cmd { + case "tensor": + // 移除子命令，让子命令处理剩余的参数 + os.Args = os.Args[2:] + tensor.Execute() + + case "version": + fmt.Printf("deepxctl 版本 %s\n", version) + + case "help": + if len(os.Args) > 2 { + helpCmd := os.Args[2] + switch helpCmd { + case "tensor": + tensor.PrintUsage() + default: + fmt.Printf("未知命令: %s\n", helpCmd) + printUsage() + } + } else { + printUsage() + } + + default: + fmt.Printf("未知命令: %s\n", cmd) + printUsage() + os.Exit(1) + } +} diff --git a/tool/deepxctl/tensor/fp16.go b/tool/deepxctl/tensor/fp16.go new file mode 100644 index 00000000..18a7e0f8 --- /dev/null +++ b/tool/deepxctl/tensor/fp16.go @@ -0,0 +1,28 @@ +package tensor + +import ( + "encoding/binary" + "math" +) + +func Byte2ToFloat16(value []byte) float32 { + bits := binary.BigEndian.Uint16(value) + // 这里需要实现float16到float32的转换 + // 简化实现，实际项目中需要更完整的实现 + sign := float32(1) + if bits&0x8000 != 0 { + sign = -1 + } + exp := int((bits & 0x7C00) >> 10) + frac := float32(bits&0x03FF) / 1024.0 + + if exp == 0 { + return sign * frac * float32(1.0/16384.0) // 非规格化数 + } else if exp == 31 { + if frac == 0 { + return sign * float32(math.Inf(1)) // 无穷大 + } + return float32(math.NaN()) // NaN + } + return sign * float32(math.Pow(2, float64(exp-15))) * (1.0 + frac) // 规格化数 +} diff --git a/tool/deepxctl/tensor/io.go b/tool/deepxctl/tensor/io.go new file mode 100644 index 00000000..b0faf9d7 --- /dev/null +++ b/tool/deepxctl/tensor/io.go @@ -0,0 +1,47 @@ +package tensor + +import ( + "encoding/binary" + "os" + + "gopkg.in/yaml.v2" +) + +func LoadShape(filePath string) (shape Shape, err error) { + var shapeData []byte + shapeData, err = os.ReadFile(filePath + ".shape") + if err != nil { + return + } + + err = yaml.Unmarshal(shapeData, &shape) + if err != nil { + return + } + return +} +func LoadTensor[T Number](filePath string) (tensor Tensor[T], err error) { + + _, err = os.ReadFile(filePath + ".shape") + if err != nil { + return + } + var shape Shape + shape, err = LoadShape(filePath) + if err != nil { + return + } + file, err := os.Open(filePath + ".data") + if err != nil { + return + } + defer file.Close() + data := make([]T, shape.Size) + + err = binary.Read(file, binary.LittleEndian, data) + if err != nil { + return + } + tensor = Tensor[T]{Data: data, Shape: shape} + return +} diff --git a/tool/deepxctl/tensor/print.go b/tool/deepxctl/tensor/print.go new file mode 100644 index 00000000..89dd3535 --- /dev/null +++ b/tool/deepxctl/tensor/print.go @@ -0,0 +1,121 @@ +package tensor + +import "fmt" + +func (t *Tensor[T]) Range(dimCount int, f func(indices []int)) { + Shape := t.Shape + if dimCount > len(Shape.Shape) { + panic("dimCount exceeds the number of dimensions in the Tensor.") + } + + totalSize := 1 + + // 计算总的循环次数 + for i := 0; i < dimCount; i++ { + totalSize *= Shape.At(i) + } + indices := make([]int, dimCount) // 初始化索引向量 + // 遍历所有可能的索引组合 + for idx := 0; idx < totalSize; idx++ { + // 反算出 indices 数组 + idx_ := idx + for dim := dimCount - 1; dim >= 0; dim-- { + indices[dim] = idx_ % Shape.At(dim) // 计算当前维度的索引 + idx_ /= Shape.At(dim) // 更新 idx + } + f(indices) // 调用传入的函数 + } +} + +func AutoFormat(dtype string) string { + switch dtype { + case "bool": + return "%v" + case "int8": + return "%d" + case "int16": + return "%d" + case "int32": + return "%d" + case "int64": + return "%d" + case "float16": + return "%f" + case "float32": + return "%f" + case "float64": + return "%f" + default: + return "%v" + } +} + +// Print 打印Tensor的值 +func (t *Tensor[T]) Print(format_ ...string) { + Shape := t.Shape + format := AutoFormat(t.Dtype) + if len(format_) > 0 { + format = format_[0] + } + fmt.Print("shape:[") + for i := 0; i < Shape.Dim; i++ { + fmt.Print(Shape.At(i)) + if i < Shape.Dim-1 { + fmt.Print(", ") + } + } + fmt.Println("]") + if Shape.Dim == 1 { + fmt.Print("[") + for i := 0; i < Shape.At(0); i++ { + if i > 0 { + fmt.Print(" ") + } + fmt.Printf(format, t.Get(i)) + } + fmt.Println("]") + } else if Shape.Dim == 2 { + fmt.Println("[") + for i := 0; i < Shape.At(0); i++ { + fmt.Print(" [") + for j := 0; j < Shape.At(1); j++ { + if j > 0 { + fmt.Print(" ") + } + fmt.Printf(format, t.Get(i, j)) + } + + fmt.Print("]") + if i < Shape.At(0)-1 { + fmt.Print(",") + } + fmt.Println() + } + fmt.Println("]") + } else { + t.Range(Shape.Dim-2, func(indices []int) { + fmt.Print(indices) + m, n := Shape.At(Shape.Dim-2), Shape.At(Shape.Dim-1) + fmt.Print([]int{m, n}) + fmt.Println("=") + + fmt.Println("[") + for i := 0; i < m; i++ { + fmt.Print(" [") + for j := 0; j < n; j++ { + if j > 0 { + fmt.Print(" ") + } + fmt.Printf(format, t.Get(append(indices, i, j)...)) + } + + fmt.Print("]") + if i < m-1 { + fmt.Print(",") + } + fmt.Println() + } + fmt.Println("]") + }) + } +} diff --git a/tool/deepxctl/tensor/tensor.go b/tool/deepxctl/tensor/tensor.go new file mode 100644 index 00000000..0ef542b3 --- /dev/null +++ b/tool/deepxctl/tensor/tensor.go @@ -0,0 +1,90 @@ +package tensor + +import ( + "fmt" +) + +type Shape struct { + Shape []int `json:"shape"` + Stride []int `json:"stride"` + Dim int `json:"ndim"` + Size int `json:"size"` + Dtype string `json:"dtype"` +} + +func NewTensorShape(shape []int) (s Shape) { + s.Dim = len(shape) + s.Shape = make([]int, len(shape)) + copy(s.Shape, shape) + s.Stride = make([]int, len(shape)) + s.Stride[len(shape)-1] = 1 + for i := len(shape) - 2; i >= 0; i-- { + s.Stride[i] = s.Stride[i+1] * shape[i+1] + } + s.Size = s.Stride[0] * shape[0] + return s +} +func (s Shape) String() string { + return fmt.Sprintf("%v", s.Shape) +} + +func (s Shape) At(i int) int { + return s.Shape[i] +} + +func (s Shape) LinearAt(indices []int) int { + idx := 0 + for i := 0; i < len(indices); i++ { + idx += indices[i] * s.Stride[i] + } + return idx +} +func (s Shape) LinearTo(idx int) (indices []int) { + linearIndex := idx + indices = make([]int, s.Dim) + for i := 0; i < s.Dim; i++ { + indices[i] = linearIndex / s.Stride[i] + linearIndex %= s.Stride[i] + } + return indices +} + +func BitSize(Dtype string) int { + switch Dtype { + case "bool": + return 8 + case "int8": + return 8 + case "int16": + return 16 + case "int32": + return 32 + case "int64": + return 64 + case "float16": + return 16 + case "float32": + return 32 + case "float64": + return 64 + default: + return 0 + } +} + +type Number interface { + comparable + float64 | float32 | int64 | int32 | int16 | int8 | bool +} + +type Tensor[T Number] struct { + Data []T + Shape +} + +// Get 获取Tensor的值 +func (t *Tensor[T]) Get(indices ...int) T { + idx := t.Shape.LinearAt(indices) + return t.Data[idx] + +} From 38acf25ccb0759c650747fce5f83d1bb54315acc Mon Sep 17 00:00:00 2001 From: lipeng <734991033@qq.com> Date: Wed, 23 Apr 2025 21:45:00 +0800 Subject: [PATCH 3/4] deepxctl --- .github/workflows/tool-deepxctl.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/tool-deepxctl.yml b/.github/workflows/tool-deepxctl.yml index a3c13ad5..f8e80fa5 100644 --- a/.github/workflows/tool-deepxctl.yml +++ b/.github/workflows/tool-deepxctl.yml @@ -34,4 +34,10 @@ jobs: run: | cd tool/deepxctl go build -v -o deepxctl + + # 运行测试 + - name: 运行测试 + run: | + cd tool/deepxctl + ./deepxctl \ No newline at end of file From c27689ae9a5cc9c4c70606a0df3b07ec631e29c6 Mon Sep 17 00:00:00 2001 From: lipeng <734991033@qq.com> Date: Wed, 23 Apr 2025 21:46:58 +0800 Subject: [PATCH 4/4] deepxctl --- tool/deepxctl/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tool/deepxctl/main.go b/tool/deepxctl/main.go index f1f72af2..1b5ffc3a 100644 --- a/tool/deepxctl/main.go +++ b/tool/deepxctl/main.go @@ -26,7 +26,7 @@ func main() { if len(os.Args) < 2 { printUsage() - os.Exit(1) + os.Exit(0) } // 获取子命令