Skip to content

Dockerfiles: run /bin/detect and /bin/build for extensions #786

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
natalieparellano wants to merge 12 commits into from
Closed

Dockerfiles: run /bin/detect and /bin/build for extensions #786

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
c0df511
WIP
Dec 13, 2021
006e371
WIP
Dec 13, 2021
a32e8e6
WIP
Dec 14, 2021
8f9a532
WIP
Dec 14, 2021
7787bec
WIP
Dec 15, 2021
8cec26a
Fix
Dec 15, 2021
1b07d9e
Rename Buildpack -> Buildable
Dec 15, 2021
ad323ee
Dockerfile has a type that is either build or run. Dockerfiles that a…
Jan 18, 2022
cb587b3
Move poc code into here
Jan 20, 2022
115e7f3
wip: Anthony + Natalie: merging in charles POC kaniko
Nov 23, 2021
d5fb974
Add script to test extender and run it for extending the build image
Jan 20, 2022
f11101e
Add comment
Jan 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
The diff you're trying to view is too large. We only load the first 3000 changed files.
40 changes: 37 additions & 3 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,9 @@ all: test build package

build: build-linux-amd64 build-linux-arm64 build-windows-amd64

build-linux-amd64: build-linux-amd64-lifecycle build-linux-amd64-symlinks build-linux-amd64-launcher
build-linux-arm64: build-linux-arm64-lifecycle build-linux-arm64-symlinks build-linux-arm64-launcher
build-windows-amd64: build-windows-amd64-lifecycle build-windows-amd64-symlinks build-windows-amd64-launcher
build-linux-amd64: build-linux-amd64-lifecycle build-linux-amd64-symlinks build-linux-amd64-launcher build-linux-amd64-extender
build-linux-arm64: build-linux-arm64-lifecycle build-linux-arm64-symlinks build-linux-arm64-launcher build-linux-arm64-extender
build-windows-amd64: build-windows-amd64-lifecycle build-windows-amd64-symlinks build-windows-amd64-launcher build-windows-amd64-extender

build-image-linux-amd64: build-linux-amd64 package-linux-amd64
build-image-linux-amd64: ARCHIVE_PATH=$(BUILD_DIR)/lifecycle-v$(LIFECYCLE_VERSION)+linux.x86-64.tgz
Expand Down Expand Up @@ -107,6 +107,28 @@ $(BUILD_DIR)/linux-arm64/lifecycle/launcher:
$(GOENV) $(GOBUILD) -o $(OUT_DIR)/launcher -a ./cmd/launcher
test $$(du -m $(OUT_DIR)/launcher|cut -f 1) -le 3

build-linux-amd64-extender: $(BUILD_DIR)/linux-amd64/lifecycle/extender

$(BUILD_DIR)/linux-amd64/lifecycle/extender: export GOOS:=linux
$(BUILD_DIR)/linux-amd64/lifecycle/extender: export GOARCH:=amd64
$(BUILD_DIR)/linux-amd64/lifecycle/extender: OUT_DIR?=$(BUILD_DIR)/$(GOOS)-$(GOARCH)/lifecycle
$(BUILD_DIR)/linux-amd64/lifecycle/extender: $(GOFILES)
$(BUILD_DIR)/linux-amd64/lifecycle/extender:
@echo "> Building lifecycle/extender for $(GOOS)/$(GOARCH)..."
mkdir -p $(OUT_DIR)
cd ./extender && $(GOENV) $(GOBUILD) -o $(OUT_DIR)/extender -a .

build-linux-arm64-extender: $(BUILD_DIR)/linux-arm64/lifecycle/extender

$(BUILD_DIR)/linux-arm64/lifecycle/extender: export GOOS:=linux
$(BUILD_DIR)/linux-arm64/lifecycle/extender: export GOARCH:=arm64
$(BUILD_DIR)/linux-arm64/lifecycle/extender: OUT_DIR?=$(BUILD_DIR)/$(GOOS)-$(GOARCH)/lifecycle
$(BUILD_DIR)/linux-arm64/lifecycle/extender: $(GOFILES)
$(BUILD_DIR)/linux-arm64/lifecycle/extender:
@echo "> Building lifecycle/extender for $(GOOS)/$(GOARCH)..."
mkdir -p $(OUT_DIR)
cd ./extender && $(GOENV) $(GOBUILD) -o $(OUT_DIR)/extender -a .

build-linux-amd64-symlinks: export GOOS:=linux
build-linux-amd64-symlinks: export GOARCH:=amd64
build-linux-amd64-symlinks: OUT_DIR?=$(BUILD_DIR)/$(GOOS)-$(GOARCH)/lifecycle
Expand Down Expand Up @@ -153,6 +175,18 @@ $(BUILD_DIR)/windows-amd64/lifecycle/launcher.exe:
@echo "> Building lifecycle/launcher for $(GOOS)/$(GOARCH)..."
$(GOBUILD) -o $(OUT_DIR)$/launcher.exe -a .$/cmd$/launcher

# ANTHONY: The following (build extender for windows) does not work
# Someone should get back to this
build-windows-amd64-extender: $(BUILD_DIR)/windows-amd64/lifecycle/extender.exe

$(BUILD_DIR)/windows-amd64/lifecycle/extender.exe: export GOOS:=windows
$(BUILD_DIR)/windows-amd64/lifecycle/extender.exe: export GOARCH:=amd64
$(BUILD_DIR)/windows-amd64/lifecycle/extender.exe: OUT_DIR?=$(BUILD_DIR)$/$(GOOS)-$(GOARCH)$/lifecycle
$(BUILD_DIR)/windows-amd64/lifecycle/extender.exe: $(GOFILES)
$(BUILD_DIR)/windows-amd64/lifecycle/extender.exe:
@echo "> Building lifecycle/extender for $(GOOS)/$(GOARCH)..."
cd .$/extender && $(GOBUILD) -o $(OUT_DIR)$/extender.exe -a .

build-windows-amd64-symlinks: export GOOS:=windows
build-windows-amd64-symlinks: export GOARCH:=amd64
build-windows-amd64-symlinks: OUT_DIR?=$(BUILD_DIR)$/$(GOOS)-$(GOARCH)$/lifecycle
Expand Down
2 changes: 1 addition & 1 deletion analyzer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ type Analyzer struct {
SBOMRestorer layer.SBOMRestorer

// Platform API < 0.7
Buildpacks []buildpack.GroupBuildpack
Buildpacks []buildpack.GroupBuildable
Cache Cache
LayerMetadataRestorer layer.MetadataRestorer
}
Expand Down
4 changes: 2 additions & 2 deletions analyzer_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ func testAnalyzerBuilder(platformAPI string) func(t *testing.T, when spec.G, it
PreviousImage: image,
Logger: &discardLogger,
Platform: p,
Buildpacks: []buildpack.GroupBuildpack{
Buildpacks: []buildpack.GroupBuildable{
{ID: "metadata.buildpack", API: api.Buildpack.Latest().String()},
{ID: "no.cache.buildpack", API: api.Buildpack.Latest().String()},
{ID: "no.metadata.buildpack", API: api.Buildpack.Latest().String()},
Expand Down Expand Up @@ -163,7 +163,7 @@ func testAnalyzerBuilder(platformAPI string) func(t *testing.T, when spec.G, it
h.AssertNil(t, testCache.SetMetadata(expectedCacheMetadata))
h.AssertNil(t, testCache.Commit())

analyzer.Buildpacks = append(analyzer.Buildpacks, buildpack.GroupBuildpack{ID: "escaped/buildpack/id", API: api.Buildpack.Latest().String()})
analyzer.Buildpacks = append(analyzer.Buildpacks, buildpack.GroupBuildable{ID: "escaped/buildpack/id", API: api.Buildpack.Latest().String()})
expectRestoresLayerMetadataIfSupported()
})

Expand Down
44 changes: 22 additions & 22 deletions builder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,11 @@ type BuildEnv interface {
List() []string
}

type BuildpackStore interface {
Lookup(bpID, bpVersion string) (buildpack.Buildpack, error)
type BuildableStore interface {
Lookup(bpID, bpVersion string) (buildpack.Buildable, error)
}

type Buildpack interface {
type Buildable interface {
Build(bpPlan buildpack.Plan, config buildpack.BuildConfig, bpEnv buildpack.BuildEnv) (buildpack.BuildResult, error)
ConfigFile() *buildpack.Descriptor
Detect(config *buildpack.DetectConfig, bpEnv buildpack.BuildEnv) buildpack.DetectRun
Expand All @@ -44,7 +44,7 @@ type Builder struct {
Plan platform.BuildPlan
Out, Err io.Writer
Logger Logger
BuildpackStore BuildpackStore
BuildableStore BuildableStore
}

func (b *Builder) Build() (*platform.BuildMetadata, error) {
Expand All @@ -66,43 +66,42 @@ func (b *Builder) Build() (*platform.BuildMetadata, error) {
var bomFiles []buildpack.BOMFile
var slices []layers.Slice
var labels []buildpack.Label
var dockerfiles []buildpack.Dockerfile

bpEnv := env.NewBuildEnv(os.Environ())
bEnv := env.NewBuildEnv(os.Environ())

for _, bp := range b.Group.Group {
b.Logger.Debugf("Running build for buildpack %s", bp)
for _, groupBuildable := range b.Group.Group { // TODO: execute extensions in parallel
b.Logger.Debugf("Running build for buildpack %s", groupBuildable)

b.Logger.Debug("Looking up buildpack")
bpTOML, err := b.BuildpackStore.Lookup(bp.ID, bp.Version)
b.Logger.Debug("Looking up buildable") // TODO: better comment
buildable, err := b.BuildableStore.Lookup(groupBuildable.ID, groupBuildable.Version)
if err != nil {
return nil, err
}

b.Logger.Debug("Finding plan")
bpPlan := plan.Find(bp.ID)
bPlan := plan.Find(groupBuildable.ID)

br, err := bpTOML.Build(bpPlan, config, bpEnv)
br, err := buildable.Build(bPlan, config, bEnv)
if err != nil {
return nil, err
}

b.Logger.Debug("Updating buildpack processes")
updateDefaultProcesses(br.Processes, api.MustParse(bp.API), b.Platform.API())

bom = append(bom, br.BOM...)
bomFiles = append(bomFiles, br.BOMFiles...)
labels = append(labels, br.Labels...)
plan = plan.Filter(br.MetRequires)

b.Logger.Debug("Updating process list")
b.Logger.Debug("Updating processes")
updateDefaultProcesses(br.Processes, api.MustParse(groupBuildable.API), b.Platform.API())
warning := processMap.add(br.Processes)
if warning != "" {
b.Logger.Warn(warning)
}

bom = append(bom, br.BOM...)
bomFiles = append(bomFiles, br.BOMFiles...)
labels = append(labels, br.Labels...)
plan = plan.Filter(br.MetRequires)
slices = append(slices, br.Slices...)
dockerfiles = append(dockerfiles, br.Dockerfiles...) // TODO: error if buildpack outputs Dockerfiles?

b.Logger.Debugf("Finished running build for buildpack %s", bp)
b.Logger.Debugf("Finished running build for buildpack %s", groupBuildable)
}

if b.Platform.API().LessThan("0.4") {
Expand All @@ -126,11 +125,12 @@ func (b *Builder) Build() (*platform.BuildMetadata, error) {
b.Logger.Debug("Finished build")
return &platform.BuildMetadata{
BOM: bom,
BuildpackDefaultProcessType: processMap.defaultType,
Buildpacks: b.Group.Group,
Dockerfiles: dockerfiles,
Labels: labels,
Processes: procList,
Slices: slices,
BuildpackDefaultProcessType: processMap.defaultType,
}, nil
}

Expand Down
40 changes: 20 additions & 20 deletions builder_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ func TestBuilder(t *testing.T) {
}

//go:generate mockgen -package testmock -destination testmock/env.go github.com/buildpacks/lifecycle BuildEnv
//go:generate mockgen -package testmock -destination testmock/buildpack_store.go github.com/buildpacks/lifecycle BuildpackStore
//go:generate mockgen -package testmock -destination testmock/buildpack_store.go github.com/buildpacks/lifecycle BuildableStore
//go:generate mockgen -package testmock -destination testmock/buildpack.go github.com/buildpacks/lifecycle Buildpack

func testBuilder(t *testing.T, when spec.G, it spec.S) {
Expand Down Expand Up @@ -71,15 +71,15 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) {
PlatformDir: platformDir,
Platform: platform.NewPlatform(api.Platform.Latest().String()),
Group: buildpack.Group{
Group: []buildpack.GroupBuildpack{
Group: []buildpack.GroupBuildable{
{ID: "A", Version: "v1", API: api.Buildpack.Latest().String(), Homepage: "Buildpack A Homepage"},
{ID: "B", Version: "v2", API: api.Buildpack.Latest().String()},
},
},
Out: stdout,
Err: stderr,
Logger: &log.Logger{Handler: logHandler},
BuildpackStore: buildpackStore,
BuildableStore: buildpackStore,
}

config, err = builder.BuildConfig()
Expand All @@ -99,7 +99,7 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) {
builder.Plan = platform.BuildPlan{
Entries: []platform.BuildPlanEntry{
{
Providers: []buildpack.GroupBuildpack{
Providers: []buildpack.GroupBuildable{
{ID: "A", Version: "v1"},
{ID: "B", Version: "v2"},
},
Expand All @@ -108,7 +108,7 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) {
},
},
{
Providers: []buildpack.GroupBuildpack{
Providers: []buildpack.GroupBuildable{
{ID: "A", Version: "v1"},
{ID: "B", Version: "v2"},
},
Expand All @@ -117,7 +117,7 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) {
},
},
{
Providers: []buildpack.GroupBuildpack{
Providers: []buildpack.GroupBuildable{
{ID: "B", Version: "v2"},
},
Requires: []buildpack.Require{
Expand Down Expand Up @@ -296,7 +296,7 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) {
when("build metadata", func() {
when("bom", func() {
it("should aggregate BOM from each buildpack", func() {
builder.Group.Group = []buildpack.GroupBuildpack{
builder.Group.Group = []buildpack.GroupBuildable{
{ID: "A", Version: "v1", API: "0.5", Homepage: "Buildpack A Homepage"},
{ID: "B", Version: "v2", API: "0.2"},
}
Expand All @@ -310,7 +310,7 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) {
Name: "dep1",
Metadata: map[string]interface{}{"version": "v1"},
},
Buildpack: buildpack.GroupBuildpack{ID: "A", Version: "v1"},
Buildpack: buildpack.GroupBuildable{ID: "A", Version: "v1"},
},
},
}, nil)
Expand All @@ -323,7 +323,7 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) {
Name: "dep2",
Metadata: map[string]interface{}{"version": "v1"},
},
Buildpack: buildpack.GroupBuildpack{ID: "B", Version: "v2"},
Buildpack: buildpack.GroupBuildable{ID: "B", Version: "v2"},
},
},
}, nil)
Expand All @@ -339,15 +339,15 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) {
Version: "",
Metadata: map[string]interface{}{"version": string("v1")},
},
Buildpack: buildpack.GroupBuildpack{ID: "A", Version: "v1"},
Buildpack: buildpack.GroupBuildable{ID: "A", Version: "v1"},
},
{
Require: buildpack.Require{
Name: "dep2",
Version: "",
Metadata: map[string]interface{}{"version": string("v1")},
},
Buildpack: buildpack.GroupBuildpack{ID: "B", Version: "v2"},
Buildpack: buildpack.GroupBuildable{ID: "B", Version: "v2"},
},
}); s != "" {
t.Fatalf("Unexpected:\n%s\n", s)
Expand All @@ -368,7 +368,7 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) {
if err != nil {
t.Fatalf("Unexpected error:\n%s\n", err)
}
if s := cmp.Diff(metadata.Buildpacks, []buildpack.GroupBuildpack{
if s := cmp.Diff(metadata.Buildpacks, []buildpack.GroupBuildable{
{ID: "A", Version: "v1", API: api.Buildpack.Latest().String(), Homepage: "Buildpack A Homepage"},
{ID: "B", Version: "v2", API: api.Buildpack.Latest().String()},
}); s != "" {
Expand Down Expand Up @@ -488,7 +488,7 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) {

when("multiple default process types", func() {
it.Before(func() {
builder.Group.Group = []buildpack.GroupBuildpack{
builder.Group.Group = []buildpack.GroupBuildable{
{ID: "A", Version: "v1", API: api.Buildpack.Latest().String()},
{ID: "B", Version: "v2", API: api.Buildpack.Latest().String()},
{ID: "C", Version: "v3", API: api.Buildpack.Latest().String()},
Expand Down Expand Up @@ -568,7 +568,7 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) {

when("overriding default process type, with a non-default process type", func() {
it.Before(func() {
builder.Group.Group = []buildpack.GroupBuildpack{
builder.Group.Group = []buildpack.GroupBuildable{
{ID: "A", Version: "v1", API: api.Buildpack.Latest().String()},
{ID: "B", Version: "v2", API: api.Buildpack.Latest().String()},
{ID: "C", Version: "v3", API: api.Buildpack.Latest().String()},
Expand Down Expand Up @@ -653,7 +653,7 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) {
when("there is a web process", func() {
when("buildpack API >= 0.6", func() {
it.Before(func() {
builder.Group.Group = []buildpack.GroupBuildpack{
builder.Group.Group = []buildpack.GroupBuildable{
{ID: "A", Version: "v1", API: api.Buildpack.Latest().String()},
}
})
Expand Down Expand Up @@ -696,7 +696,7 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) {

when("buildpack api < 0.6", func() {
it.Before(func() {
builder.Group.Group = []buildpack.GroupBuildpack{
builder.Group.Group = []buildpack.GroupBuildable{
{ID: "A", Version: "v1", API: "0.5"},
}
})
Expand Down Expand Up @@ -841,7 +841,7 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) {
Name: "dep1",
Metadata: map[string]interface{}{"version": string("v1")},
},
Buildpack: buildpack.GroupBuildpack{ID: "A", Version: "v1"},
Buildpack: buildpack.GroupBuildable{ID: "A", Version: "v1"},
},
},
}, nil)
Expand All @@ -861,7 +861,7 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) {
Version: "v1",
Metadata: map[string]interface{}{"version": string("v1")},
},
Buildpack: buildpack.GroupBuildpack{ID: "A", Version: "v1"},
Buildpack: buildpack.GroupBuildable{ID: "A", Version: "v1"},
},
}); s != "" {
t.Fatalf("Unexpected:\n%s\n", s)
Expand All @@ -879,7 +879,7 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) {
when("there is a web process", func() {
when("buildpack API >= 0.6", func() {
it.Before(func() {
builder.Group.Group = []buildpack.GroupBuildpack{
builder.Group.Group = []buildpack.GroupBuildable{
{ID: "A", Version: "v1", API: api.Buildpack.Latest().String()},
}
})
Expand Down Expand Up @@ -923,7 +923,7 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) {

when("buildpack api < 0.6", func() {
it.Before(func() {
builder.Group.Group = []buildpack.GroupBuildpack{
builder.Group.Group = []buildpack.GroupBuildable{
{ID: "A", Version: "v1", API: "0.5"},
}
})
Expand Down
Loading