diff --git a/hack/gen-release-docs.sh b/hack/gen-release-docs.sh
index ce791435ce..b9bce3fb86 100755
--- a/hack/gen-release-docs.sh
+++ b/hack/gen-release-docs.sh
@@ -31,7 +31,8 @@ CONTENT_DIR=docs/content/en
 # Create new $CONTENT_DIR/docs-$VERSION
 rm -rf $CONTENT_DIR/docs-$VERSION
 cp -rf $CONTENT_DIR/docs-dev $CONTENT_DIR/docs-$VERSION
-cp -rf docs/themes/docsy/layouts/docs/ docs/layouts/docs-$VERSION
+rm -rf docs/layouts/docs-$VERSION
+cp -rf docs/themes/docsy/layouts/docs docs/layouts/docs-$VERSION
 cat <<EOT > $CONTENT_DIR/docs-$VERSION/_index.md
 ---
 title: "Welcome to PipeCD"