Refactor Build Process: Use pokesprite as submodule

.github/workflows/main.yml

@@ -4,8 +4,10 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v2
+      - name: Checkout 
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
       - name: Log in to the Container registry
         uses: docker/login-action@v1
         with:
@@ -16,16 +18,18 @@ jobs:
         run: docker pull ghcr.io/tmck-code/pokesay:latest
       - name: Make dist directory
         run: mkdir -p dist/bin/ dist/packages/ dist/tarballs/
+      - name: Build cows
+        run: docker run -v ${PWD}:/usr/local/src -v ${PWD}/build/cows:/tmp/cows -v ${PWD}/build/pokesprite:/tmp/original/pokesprite -u root --platform linux/amd64 ghcr.io/tmck-code/pokesay:latest ./build/scripts/build_cowfiles.sh
       - name: Build go binary assets
-        run: docker run -v ${PWD}:/usr/local/src -u root --platform linux/amd64 -e VERSION=0.1 ghcr.io/tmck-code/pokesay:latest ./build/scripts/build_assets.sh
+        run: docker run -v ${PWD}:/usr/local/src -v ${PWD}/build/cows:/tmp/cows -u root --platform linux/amd64 -e VERSION=0.1 ghcr.io/tmck-code/pokesay:latest ./build/scripts/build_assets.sh
+      - name: Run unit tests
+        run: docker run -v ${PWD}:/usr/local/src -u root ghcr.io/tmck-code/pokesay:latest gotestsum --debug
       - name: Build go binaries
         run: docker run -v ${PWD}:/usr/local/src -u root --platform linux/amd64 -e VERSION=0.1 ghcr.io/tmck-code/pokesay:latest ./build/scripts/build_bin.sh
       - name: Check build
         run: ls -alh dist/bin
       - name: Test build
         run: echo w | ./dist/bin/pokesay-0.1-linux-amd64
-      - name: Run unit tests
-        run: docker run -v ${PWD}:/usr/local/src -u root ghcr.io/tmck-code/pokesay:latest gotestsum --debug
       - name: Build deb package
         run: docker run -v ${PWD}:/usr/local/src -u root -e VERSION=0.1 -e DEBUG=1 ghcr.io/tmck-code/pokesay:latest bash -c "./build/scripts/build_packages.sh deb"
       - name: Build arch package

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "build/pokesprite"]
+	path = build/pokesprite
+	url = git@github.com:msikma/pokesprite

build/Dockerfile

@@ -15,30 +15,15 @@ RUN apt-get update \
     && rm -rf /var/lib/apt/lists/* \
     && rm -rf /tmp/* /var/tmp/*
 
+
+RUN useradd u -m
+
 WORKDIR /usr/local/src/
-ADD src/ /usr/local/src/src/
 ADD go.* /usr/local/src/
+ADD src/ /usr/local/src/src/
 RUN go mod tidy \
     && go get -v github.com/mitchellh/go-wordwrap \
-    && go install gotest.tools/gotestsum@latest
-
-RUN useradd u -m
-USER u
-
-RUN git clone -q --depth 1 https://github.com/msikma/pokesprite /tmp/original/pokesprite
-
-ARG DEBUG
-ENV DEBUG=$DEBUG
-RUN echo "Debug mode is set to: '$DEBUG'"
-# Convert all of the pokesprite .pngs -> cowfiles for the terminal
-RUN go run /usr/local/src/src/bin/convert/png_convert.go \
-        -from /tmp/original/pokesprite/ \
-        -to /tmp/cows/ \
-        -padding 4 \
-        -skip '["resources/", "misc/", "icons/", "items/", "items-outline/"]' \
-    && mv -v /tmp/cows/pokemon-gen8 /tmp/cows/gen8 \
-    && mv -v /tmp/cows/pokemon-gen7x /tmp/cows/gen7x \
-    && cat /tmp/original/pokesprite/data/pokemon.json | jq -c .[] > /tmp/cows/pokemon.json \
-    && rm -rf /tmp/original/pokesprite
+    && go install gotest.tools/gotestsum@latest \
+    && chown -R u:u /go
 
 ADD . .

build/Makefile

@@ -29,23 +29,29 @@ build/docker:
 build/cows:
 	@$(echo) -e "\e[48;5;30m> Building cows\e[0m"
 	@rm -rf cows.tar.gz cows/
-	@docker rm -f pokebuilder > /dev/null 2>&1
-	@docker create \
+	@mkdir -p cows/
+	@docker run --rm \
 		--platform linux/amd64 \
-		--name pokebuilder $(DOCKER_IMAGE)
-	docker cp pokebuilder:$(DOCKER_OUTPUT_DIR)/ cows/
-	@tar czf cows.tar.gz cows/
-	@rm -rf cows/
-	@docker rm -f pokebuilder > /dev/null 2>&1
+		-v $(PWD)/../:/usr/local/src \
+		-v $(PWD)/pokesprite:/tmp/original/pokesprite \
+		-v $(PWD)/cows:/tmp/cows \
+		-u u \
+		$(DOCKER_IMAGE) \
+		  build/scripts/build_cowfiles.sh
+	@tar -czf cows.tar.gz ./cows
 	@du -sh cows.tar.gz
 
 # generate embedded bin files for category/metadata/the actual pokemon
 build/assets:
 	@$(echo) -e "\e[48;5;30m> Building assets\e[0m"
 	@mkdir -p $(PWD)/assets
+	@rm -rf $(PWD)/cows/
+	@tar -xzf cows.tar.gz
 	@docker run --rm \
 		-v $(PWD)/../:/usr/local/src \
+		-v $(PWD)/cows:/tmp/cows \
 		--platform linux/amd64 \
+		-u u \
 		--name pokesay \
 	    $(DOCKER_IMAGE) \
 			build/scripts/build_assets.sh
@@ -59,7 +65,7 @@ build/bin:
 		-e VERSION=$(VERSION) \
 		-e DEBUG=$(DEBUG) \
 		--platform linux/amd64 \
-		--user root \
+		--user u \
 		--name pokesay \
 		  $(DOCKER_IMAGE) \
 		  /usr/local/src/build/scripts/build_bin.sh

build/scripts/build_cowfiles.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -euo pipefail
+
+mkdir -p /tmp/convert/
+
+# Convert all of the pokesprite .pngs -> cowfiles for the terminal
+go run /usr/local/src/src/bin/convert/png_convert.go \
+  -from /tmp/original/pokesprite/ \
+  -tmpDir /tmp/convert/ \
+  -to /tmp/cows/ \
+  -padding 4 \
+  -skip '["resources/", "misc/", "icons/", "items/", "items-outline/"]' \
+  && mv -v /tmp/cows/pokemon-gen8 /tmp/cows/gen8 > /dev/null \
+  && mv -v /tmp/cows/pokemon-gen7x /tmp/cows/gen7x > /dev/null \
+  && cat /tmp/original/pokesprite/data/pokemon.json | jq -c .[] > /tmp/cows/pokemon.json

src/bin/convert/png_convert.go

@@ -6,7 +6,12 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
+	"sync"
+	"time"
+
+	"github.com/schollz/progressbar/v3"
 
 	"github.com/tmck-code/pokesay/src/bin"
 	"github.com/tmck-code/pokesay/src/pokedex"
@@ -18,6 +23,7 @@ var (
 
 type CowBuildArgs struct {
 	FromDir  string
+	TmpDir   string
 	ToDir    string
 	SkipDirs []string
 	Padding  int
@@ -26,13 +32,14 @@ type CowBuildArgs struct {
 
 func parseArgs() CowBuildArgs {
 	fromDir := flag.String("from", ".", "from dir")
+	tmpDir := flag.String("tmpDir", "/tmp/convert/", "temporary directory for intermediate files")
 	toDir := flag.String("to", ".", "to dir")
 	skipDirs := flag.String("skip", "'[\"resources\"]'", "JSON array of dir patterns to skip converting")
 	padding := flag.Int("padding", 2, "the number of spaces to pad from the left")
 
 	flag.Parse()
 
-	args := CowBuildArgs{FromDir: *fromDir, ToDir: *toDir, Padding: *padding, Debug: DEBUG}
+	args := CowBuildArgs{FromDir: *fromDir, TmpDir: *tmpDir, ToDir: *toDir, Padding: *padding, Debug: DEBUG}
 	json.Unmarshal([]byte(*skipDirs), &args.SkipDirs)
 
 	if args.Debug {
@@ -41,44 +48,32 @@ func parseArgs() CowBuildArgs {
 	return args
 }
 
-func main() {
-	args := parseArgs()
-
-	fpaths := pokedex.FindFiles(args.FromDir, ".png", args.SkipDirs)
-
-	// Ensure that the destination dir exists
-	os.MkdirAll(args.ToDir, 0755)
-
-	fmt.Println("Converting PNGs -> cowfiles")
-	pbar := bin.NewProgressBar(len(fpaths))
+func worker(args CowBuildArgs, jobs <-chan string, pbar *progressbar.ProgressBar, dataSet map[string]struct{}, nDuplicates *int, nFailures *int, mu *sync.Mutex, wg *sync.WaitGroup) {
+	defer wg.Done()
 
-	allData := make([]string, 0, len(fpaths))
-	nDuplicates, nFailures := 0, 0
+	for f := range jobs {
+		data, err := pokedex.ConvertPngToCow(args.FromDir, f, args.TmpDir, args.ToDir, args.Padding)
+		pbar.Add(1)
 
-	for _, f := range fpaths {
-		data, err := pokedex.ConvertPngToCow(args.FromDir, f, args.ToDir, args.Padding)
 		if err != nil {
-			nFailures++
+			mu.Lock()
+			*nFailures++
+			mu.Unlock()
 			continue
 		}
 
 		// check if this cawfile is a duplicate of one that has already been written
-		found := false
-		for _, existingData := range allData {
-			if existingData == data {
-				found = true
-				break
-			}
-		}
-		if found {
+		mu.Lock()
+		if _, found := dataSet[data]; found {
 			if args.Debug {
 				fmt.Println("Skipping duplicate data for", f)
 			}
-			nDuplicates++
-			pbar.Add(1)
+			*nDuplicates++
+			mu.Unlock()
 			continue
 		}
-		allData = append(allData, data)
+		dataSet[data] = struct{}{}
+		mu.Unlock()
 
 		destDirpath := filepath.Join(
 			args.ToDir,
@@ -89,8 +84,54 @@ func main() {
 		destFpath := filepath.Join(destDirpath, strings.ReplaceAll(filepath.Base(f), ".png", ".cow"))
 
 		pokedex.WriteToCowfile(data, destDirpath, destFpath)
-		pbar.Add(1)
 	}
+}
+
+func main() {
+	args := parseArgs()
+
+	fpaths := pokedex.FindFiles(args.FromDir, ".png", args.SkipDirs)
+
+	// Ensure that the destination dir exists
+	os.MkdirAll(args.ToDir, 0755)
+
+	fmt.Println("Converting PNGs -> cowfiles")
+	pbar := bin.NewProgressBar(len(fpaths))
+
+	dataSet := make(map[string]struct{})
+	nDuplicates, nFailures := 0, 0
+
+	nWorkers := runtime.NumCPU()
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+
+	// Create a channel to distribute work
+	jobs := make(chan string, len(fpaths))
+
+	// Send all file paths to the jobs channel
+	for _, f := range fpaths {
+		jobs <- f
+	}
+	close(jobs)
+
+	// Start worker goroutines
+	for w := 0; w < nWorkers; w++ {
+		wg.Add(1)
+		go worker(args, jobs, &pbar, dataSet, &nDuplicates, &nFailures, &mu, &wg)
+	}
+
+	// Wait for all workers to finish
+	wg.Wait()
 	fmt.Println("\nFinished converting", len(fpaths), "pokesprite PNGs -> cowfiles")
 	fmt.Println("(skipped", nDuplicates, "duplicates and", nFailures, "failures)")
+
+	// wait for progress bar to finish
+	time.Sleep(100 * time.Millisecond)
+
+	if args.Debug && len(pokedex.Failures) > 0 {
+		fmt.Println("failures:")
+		for _, f := range pokedex.Failures {
+			fmt.Println(" -", f)
+		}
+	}
 }