From b52cdffb49abd5e84ea529434159882b8b03c118 Mon Sep 17 00:00:00 2001 From: Fake User Date: Mon, 6 Jan 2025 14:46:09 +0100 Subject: [PATCH 1/4] Post-release version bump to 2.9 --- NEWS | 8 ++++++++ configure.ac | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 1943f35..c77388b 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,11 @@ +Changes in version 2.9 +====================== + +STILL UNDER DEVELOPMENT; NOT RELEASED YET. + +* No changes recorded. + + Changes in version 2.8 ====================== diff --git a/configure.ac b/configure.ac index 4385a7c..444324f 100644 --- a/configure.ac +++ b/configure.ac @@ -26,7 +26,7 @@ dnl THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT dnl (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE dnl OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -AC_INIT([NetBSD System Builder], [2.8], +AC_INIT([NetBSD System Builder], [2.9], [https://github.com/jmmv/sysbuild/issues/], [sysbuild], []) AC_PREREQ([2.65]) From 2c032fd2cc45e235c1b22c4fa9892a3f63f38854 Mon Sep 17 00:00:00 2001 From: Fake User Date: Mon, 6 Jan 2025 14:47:14 +0100 Subject: [PATCH 2/4] Add support for git Leverage the new git module in shtk-1.8 to support Git clones and updates in addition to CVS. CVS remains the default SCM for backwards compatibility reasons, but this is subject to change. --- NEWS | 4 +- NetBSD.conf | 1 + configure.ac | 2 +- default.conf | 9 +-- sysbuild.conf.5 | 29 +++++++- sysbuild.sh | 56 +++++++++++---- sysbuild_test.sh | 177 ++++++++++++++++++++++++++++++++++++++++++++--- 7 files changed, 251 insertions(+), 27 deletions(-) diff --git a/NEWS b/NEWS index c77388b..fba88ab 100644 --- a/NEWS +++ b/NEWS @@ -3,7 +3,9 @@ Changes in version 2.9 STILL UNDER DEVELOPMENT; NOT RELEASED YET. -* No changes recorded. +* Added support to fetch sources from Git in addition to CVS. The new + SCM variable controls which system to use and still defaults to CVS. + The new GIT_{BRANCH,REPO}_{SRC,XSRC} variables configure Git access. Changes in version 2.8 diff --git a/NetBSD.conf b/NetBSD.conf index 1ae4847..8623a13 100644 --- a/NetBSD.conf +++ b/NetBSD.conf @@ -25,6 +25,7 @@ BUILD_TARGETS="release" INCREMENTAL_BUILD="no" # Determine the CVS root and the current release. +SCM="git" if [ -f "${SRCDIR}/CVS/Root" ]; then CVSROOT="$(cat ${SRCDIR}/CVS/Root)" else diff --git a/configure.ac b/configure.ac index 444324f..b63bda7 100644 --- a/configure.ac +++ b/configure.ac @@ -65,7 +65,7 @@ m4_ifndef([PKG_CHECK_MODULES], m4_ifndef([SHTK_CHECK], [m4_fatal([Cannot find shtk.m4; see the INSTALL document for help])]) -SHTK_CHECK([>= 1.7]) +SHTK_CHECK([>= 1.8]) AM_CONDITIONAL([HOST_IS_NETBSD], [test "$(uname -s)" = NetBSD]) diff --git a/default.conf b/default.conf index 90e989e..4f9ff75 100644 --- a/default.conf +++ b/default.conf @@ -5,7 +5,7 @@ # # This file does not assume that the host is NetBSD. For that reason, # it cannot automatically determine "valid" settings for some variables -# like MACHINES or CVSTAG and therefore only generic values are set. +# like MACHINES or GIT_*_BRANCH and therefore only generic values are set. # Place all build files in the user's home directory. BUILD_ROOT="${HOME}/netbsd" @@ -26,6 +26,7 @@ BUILD_TARGETS="release" # Do not perform update builds by default. INCREMENTAL_BUILD="no" -# Set the CVS root and the desired release. -CVSROOT=":ext:anoncvs@anoncvs.NetBSD.org:/cvsroot" -#CVSTAG="netbsd-6" +# Set the Git repo and the desired release. +SCM="git" +GIT_SRC_REPO="https://github.com/NetBSD/src.git" +#GIT_SRC_BRANCH="netbsd-6" diff --git a/sysbuild.conf.5 b/sysbuild.conf.5 index ce98eb1..5c910ae 100644 --- a/sysbuild.conf.5 +++ b/sysbuild.conf.5 @@ -25,7 +25,7 @@ .\" THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT .\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE .\" OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Dd December 31, 2024 +.Dd January 6, 2025 .Dt SYSBUILD.CONF 5 .Os .Sh NAME @@ -52,6 +52,33 @@ Default: CVS tag to use during checkouts or updates of the src and xsrc modules. .Pp Default: not defined. +.It Va GIT_SRC_BRANCH +Name of the Git branch to check out for the src module. +Default: +.Sq trunk . +.It Va GIT_SRC_REPO +URI of the Git repo that contains the src module. +Default: +.Sq https://github.com/NetBSD/src.git . +.It Va GIT_XSRC_BRANCH +Name of the Git branch to check out for the xsrc module. +Default: +.Sq trunk . +.It Va GIT_XSRC_REPO +URI of the Git repo that contains the xsrc module. +Default: +.Sq https://github.com/NetBSD/xsrc.git . +.It Va SCM +Name of the version control system to use. +Can be one of: +.Sq cvs +or +.Sq git . +.Pp +Default: +.Sq cvs . +This will change in the future as CVS is phased out as the preferred SCM for +NetBSD development. .It Va SRCDIR Path to the src module. If you want diff --git a/sysbuild.sh b/sysbuild.sh index 9dece9a..1dd45ff 100644 --- a/sysbuild.sh +++ b/sysbuild.sh @@ -32,6 +32,7 @@ shtk_import cli shtk_import config shtk_import cvs +shtk_import git shtk_import hw shtk_import list shtk_import process @@ -40,9 +41,12 @@ shtk_import process # List of valid configuration variables. # # Please remember to update sysbuild.conf(5) if you change this list. -SYSBUILD_CONFIG_VARS="BUILD_ROOT BUILD_TARGETS CVSROOT CVSTAG INCREMENTAL_BUILD - MACHINES MKVARS NJOBS RELEASEDIR SRCDIR UPDATE_SOURCES - XSRCDIR" +SYSBUILD_CONFIG_VARS=" + BUILD_ROOT BUILD_TARGETS CVSROOT CVSTAG + GIT_SRC_BRANCH GIT_SRC_REPO GIT_XSRC_BRANCH GIT_XSRC_REPO + INCREMENTAL_BUILD MACHINES MKVARS NJOBS RELEASEDIR + SCM SRCDIR + UPDATE_SOURCES XSRCDIR" # Paths to installed files. @@ -62,10 +66,15 @@ sysbuild_set_defaults() { shtk_config_set BUILD_ROOT "${HOME}/sysbuild" shtk_config_set BUILD_TARGETS "release" shtk_config_set CVSROOT ":ext:anoncvs@anoncvs.NetBSD.org:/cvsroot" + shtk_config_set GIT_SRC_BRANCH "trunk" + shtk_config_set GIT_SRC_REPO "https://github.com/NetBSD/src.git" + shtk_config_set GIT_XSRC_BRANCH "trunk" + shtk_config_set GIT_XSRC_REPO "https://github.com/NetBSD/xsrc.git" shtk_config_set INCREMENTAL_BUILD "false" shtk_config_set MACHINES "$(uname -m)" shtk_config_set NJOBS "$(shtk_hw_ncpus)" shtk_config_set RELEASEDIR "${HOME}/sysbuild/release" + shtk_config_set SCM "cvs" shtk_config_set SRCDIR "${HOME}/sysbuild/src" shtk_config_set UPDATE_SOURCES "true" @@ -295,17 +304,40 @@ sysbuild_fetch() { shtk_config_run_hook pre_fetch_hook - local cvsroot="$(shtk_config_get CVSROOT)" + local scm="$(shtk_config_get SCM)" + case "${scm}" in + cvs) + local cvsroot="$(shtk_config_get CVSROOT)" - shtk_cli_info "Updating base source tree" - shtk_cvs_fetch "${cvsroot}" src "$(shtk_config_get_default CVSTAG '')" \ - "$(shtk_config_get SRCDIR)" + shtk_cli_info "Updating base source tree" + shtk_cvs_fetch "${cvsroot}" src "$(shtk_config_get_default CVSTAG '')" \ + "$(shtk_config_get SRCDIR)" - if shtk_config_has XSRCDIR; then - shtk_cli_info "Updating X11 source tree" - shtk_cvs_fetch "${cvsroot}" xsrc \ - "$(shtk_config_get_default CVSTAG '')" "$(shtk_config_get XSRCDIR)" - fi + if shtk_config_has XSRCDIR; then + shtk_cli_info "Updating X11 source tree" + shtk_cvs_fetch "${cvsroot}" xsrc \ + "$(shtk_config_get_default CVSTAG '')" "$(shtk_config_get XSRCDIR)" + fi + ;; + + git) + local srcrepo="$(shtk_config_get GIT_SRC_REPO)" + local srcbranch="$(shtk_config_get GIT_SRC_BRANCH)" + shtk_cli_info "Updating base source tree" + shtk_git_fetch "${srcrepo}" "${srcbranch}" "$(shtk_config_get SRCDIR)" + + if shtk_config_has XSRCDIR; then + local xsrcrepo="$(shtk_config_get GIT_XSRC_REPO)" + local xsrcbranch="$(shtk_config_get GIT_XSRC_BRANCH)" + shtk_cli_info "Updating X11 source tree" + shtk_git_fetch "${xsrcrepo}" "${xsrcbranch}" "$(shtk_config_get XSRCDIR)" + fi + ;; + + *) + shtk_cli_error "Unknown SCM type ${scm}" + ;; + esac shtk_config_run_hook post_fetch_hook } diff --git a/sysbuild_test.sh b/sysbuild_test.sh index 5561f07..290f228 100644 --- a/sysbuild_test.sh +++ b/sysbuild_test.sh @@ -95,6 +95,49 @@ create_mock_cvsroot() { } +# Creates a fake Git repository for the src module. +# +# \param dir Path to the src repository to create. +# \param branch Name of the branch to create. +create_mock_git_src() { + local dir="${1}"; shift + local branch="${1}"; shift + + assert_command -o ignore -e ignore git init --bare -b "${branch}" "${dir}" + git config --global user.email fake@example.com + git config --global user.name "Fake User" + + assert_command -o ignore -e ignore git clone "${dir}" tmp + create_mock_binary tmp/build.sh + echo "first revision" >tmp/file-in-src + assert_command -o ignore -e ignore -w tmp git add build.sh file-in-src + assert_command -o ignore -e ignore -w tmp git commit -a -m 'Import' + assert_command -o ignore -e ignore -w tmp git push + rm -rf tmp +} + + +# Creates a fake Git repository for the xsrc module. +# +# \param dir Path to the xsrc repository to create. +# \param branch Name of the branch to create. +create_mock_git_xsrc() { + local dir="${1}"; shift + local branch="${1}"; shift + + assert_command -o ignore -e ignore git init --bare -b "${branch}" "${dir}" + git config --global user.email fake@example.com + git config --global user.name "Fake User" + + assert_command -o ignore -e ignore git clone "${dir}" tmp + echo "first revision" >tmp/file-in-xsrc + assert_command -o ignore -e ignore -w tmp git add file-in-xsrc + assert_command -o ignore -e ignore -w tmp git commit -a -m 'Import' + assert_command -o ignore -e ignore -w tmp git push + rm -rf tmp +} + + shtk_unittest_add_test build__custom_dirs build__custom_dirs_test() { mock_cvsroot=":local:$(pwd)/cvsroot" @@ -734,11 +777,16 @@ BUILD_ROOT = ${HOME}/sysbuild BUILD_TARGETS = release CVSROOT = :ext:anoncvs@anoncvs.NetBSD.org:/cvsroot CVSTAG is undefined +GIT_SRC_BRANCH = trunk +GIT_SRC_REPO = https://github.com/NetBSD/src.git +GIT_XSRC_BRANCH = trunk +GIT_XSRC_REPO = https://github.com/NetBSD/xsrc.git INCREMENTAL_BUILD = false MACHINES = $(uname -m) MKVARS is undefined NJOBS = 123 RELEASEDIR = ${HOME}/sysbuild/release +SCM = cvs SRCDIR = ${HOME}/sysbuild/src UPDATE_SOURCES = true XSRCDIR is undefined @@ -819,11 +867,16 @@ BUILD_ROOT = /tmp/test BUILD_TARGETS = release CVSROOT = foo bar CVSTAG = the-new-tag +GIT_SRC_BRANCH = trunk +GIT_SRC_REPO = https://github.com/NetBSD/src.git +GIT_XSRC_BRANCH = trunk +GIT_XSRC_REPO = https://github.com/NetBSD/xsrc.git INCREMENTAL_BUILD = false MACHINES = $(uname -m) MKVARS is undefined NJOBS = 88 RELEASEDIR = ${HOME}/sysbuild/release +SCM = cvs SRCDIR is undefined UPDATE_SOURCES = true XSRCDIR is undefined @@ -982,8 +1035,8 @@ EOF } -shtk_unittest_add_test fetch__checkout__src_only -fetch__checkout__src_only_test() { +shtk_unittest_add_test fetch__cvs__checkout__src_only +fetch__cvs__checkout__src_only_test() { mock_cvsroot=":local:$(pwd)/cvsroot" create_mock_cvsroot "${mock_cvsroot}" cat >test.conf <test.conf <test.conf <test.conf <test.conf <test.conf <test.conf <tmp/file-in-src + assert_command -o ignore -e ignore -w tmp git commit -a -m 'Second revision' + assert_command -o ignore -e ignore -w tmp git push + rm -rf tmp + + test -f checkout/src/file-in-src || fail "src not present yet" + if grep "second revision" checkout/src/file-in-src >/dev/null; then + fail "second revision already present" + fi + + assert_command -o ignore -e not-match:"xsrc" sysbuild -c test.conf fetch + + grep "second revision" checkout/src/file-in-src >/dev/null \ + || fail "src not updated" + test ! -d checkout/xsrc || fail "xsrc checked out but not requested" +} + + +shtk_unittest_add_test fetch__git__update__src_and_xsrc +fetch__git__update__src_and_xsrc_test() { + create_mock_git_src "$(pwd)/src.git" trunk + create_mock_git_xsrc "$(pwd)/xsrc.git" trunk + cat >test.conf <"tmp/file-in-$m" + assert_command -o ignore -e ignore -w tmp git commit -a -m 'Second revision' + assert_command -o ignore -e ignore -w tmp git push + rm -rf tmp + done + + test -f checkout/src/file-in-src || fail "src not present yet" + if grep "second revision" checkout/src/file-in-src >/dev/null; then + fail "second revision already present" + fi + test -f checkout/xsrc/file-in-xsrc || fail "xsrc not present yet" + if grep "second revision" checkout/xsrc/file-in-xsrc >/dev/null; then + fail "second revision already present" + fi + + assert_command -o ignore -e ignore sysbuild -c test.conf fetch + + grep "second revision" checkout/src/file-in-src >/dev/null \ + || fail "src not updated" + grep "second revision" checkout/xsrc/file-in-xsrc >/dev/null \ + || fail "xsrc not updated" +} + + shtk_unittest_add_test fetch__hooks__ok fetch__hooks__ok_test() { mock_cvsroot=":local:$(pwd)/cvsroot" From 4d3212de3cb634b1511341d7b82ac85c9b3c5933 Mon Sep 17 00:00:00 2001 From: Julio Merino Date: Tue, 7 Jan 2025 19:39:43 -0800 Subject: [PATCH 3/4] Error out on fetch operations on CVS/Git mismatch Now that we support two SCMs, make sure we don't use the wrong one on an existing copy of the src and xsrc checkouts. This will avoid some user errors. --- sysbuild.sh | 33 +++++++++++++++++++--- sysbuild_test.sh | 72 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 4 deletions(-) diff --git a/sysbuild.sh b/sysbuild.sh index 1dd45ff..46ed510 100644 --- a/sysbuild.sh +++ b/sysbuild.sh @@ -309,28 +309,53 @@ sysbuild_fetch() { cvs) local cvsroot="$(shtk_config_get CVSROOT)" + local srcdir="$(shtk_config_get SRCDIR)" + if [ -d "${srcdir}" -a -e "${srcdir}/.git" ]; then + shtk_cli_error "SCM=${scm} but ${srcdir} looks" \ + "like a Git repo" + fi + shtk_cli_info "Updating base source tree" shtk_cvs_fetch "${cvsroot}" src "$(shtk_config_get_default CVSTAG '')" \ - "$(shtk_config_get SRCDIR)" + "${srcdir}" if shtk_config_has XSRCDIR; then + local xsrcdir="$(shtk_config_get XSRCDIR)" + if [ -d "${xsrcdir}" -a -e "${xsrcdir}/.git" ]; then + shtk_cli_error "SCM=${scm} but ${xsrcdir} looks" \ + "like a Git repo" + fi + shtk_cli_info "Updating X11 source tree" shtk_cvs_fetch "${cvsroot}" xsrc \ - "$(shtk_config_get_default CVSTAG '')" "$(shtk_config_get XSRCDIR)" + "$(shtk_config_get_default CVSTAG '')" "${xsrcdir}" fi ;; git) local srcrepo="$(shtk_config_get GIT_SRC_REPO)" local srcbranch="$(shtk_config_get GIT_SRC_BRANCH)" + + local srcdir="$(shtk_config_get SRCDIR)" + if [ -d "${srcdir}" -a -e "${srcdir}/CVS" ]; then + shtk_cli_error "SCM=${scm} but ${srcdir} looks" \ + "like a CVS checkout" + fi + shtk_cli_info "Updating base source tree" - shtk_git_fetch "${srcrepo}" "${srcbranch}" "$(shtk_config_get SRCDIR)" + shtk_git_fetch "${srcrepo}" "${srcbranch}" "${srcdir}" if shtk_config_has XSRCDIR; then + local xsrcdir="$(shtk_config_get XSRCDIR)" + if [ -d "${xsrcdir}" -a -e "${xsrcdir}/CVS" ]; then + shtk_cli_error "SCM=${scm} but ${xsrcdir} looks" \ + "like a CVS checkout" + fi + local xsrcrepo="$(shtk_config_get GIT_XSRC_REPO)" local xsrcbranch="$(shtk_config_get GIT_XSRC_BRANCH)" shtk_cli_info "Updating X11 source tree" - shtk_git_fetch "${xsrcrepo}" "${xsrcbranch}" "$(shtk_config_get XSRCDIR)" + shtk_git_fetch "${xsrcrepo}" "${xsrcbranch}" "${xsrcdir}" fi ;; diff --git a/sysbuild_test.sh b/sysbuild_test.sh index 290f228..151a727 100644 --- a/sysbuild_test.sh +++ b/sysbuild_test.sh @@ -1147,6 +1147,78 @@ EOF } +shtk_unittest_add_test fetch__cvs__update__src_is_git +fetch__cvs__update__src_is_git_test() { + mkdir -p checkout/src/.git + cat >test.conf <experr <test.conf <test.conf <experr <test.conf < Date: Tue, 7 Jan 2025 19:45:33 -0800 Subject: [PATCH 4/4] Use shtk from HEAD during GitHub testing We need the future shtk 1.8 but it hasn't been released yet, so pull shtk from Git for now. I'll revert back to 1.8 once a new sysbuild release is ready to be cut. --- .github/workflows/install-shtk.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/install-shtk.sh b/.github/workflows/install-shtk.sh index ca9d68e..f4737b5 100755 --- a/.github/workflows/install-shtk.sh +++ b/.github/workflows/install-shtk.sh @@ -28,7 +28,7 @@ set -eux -readonly VERSION=1.7 +readonly VERSION=HEAD main() { local prefix="${1}"; shift @@ -39,9 +39,15 @@ main() { cd "${tmpdir}" - curl -LO "https://github.com/jmmv/shtk/releases/download/shtk-${VERSION}/shtk-${VERSION}.tar.gz" - tar xzvf "shtk-${VERSION}.tar.gz" - cd "shtk-${VERSION}" + if [ "${VERSION}" = HEAD ]; then + git clone https://github.com/jmmv/shtk.git + cd shtk + autoreconf -is + else + curl -LO "https://github.com/jmmv/shtk/releases/download/shtk-${VERSION}/shtk-${VERSION}.tar.gz" + tar xzvf "shtk-${VERSION}.tar.gz" + cd "shtk-${VERSION}" + fi ./configure --prefix="${prefix}" make sudo make install