[FEATURE REQUEST] - Support for including external bcl files

[oarkflow]

It would be great if this library also support external bcl files using some expressions like. include, import

[wkhere]

I agree and it's on my radar together with functions; maybe even earlier. There are couple of things to consider, like: import as some prefixed scope or include it flattening to the main scope; I'm preferring the latter a bit, althought it would require a need to embed file info together with positions in Prog struct - an implementation detail you don't need to worry about. There's a short note about it ("Reading multiple files") and part of the job is done since v0.10.5 - that is properly processing multiple inputs by a lexer.

[oarkflow]

@wkhere Yeah I was also thinking about the later option to flattening to main scope. I've forked your repo and added support for

1. env
2. include

package main

import (
	"fmt"
	"os"
	"strings"

)

var cfg = []byte(`@include "credentials.bcl"
var domain = username+"@acme.com"
var appName = "{{env.APP_NAME:'dev'}}"
var default_port    = 8400
var local_port_base = default_port + 1000

def tunnel "myservice-prod1" {
	host = appName + "." + domain
	local_port  = local_port_base + 1
	remote_port = default_port
	enabled = true

	def extras {
		max_latency = 8.5 # [ms]
	}
}

def tunnel "myservice-prod2" {
	host = appName + "." + domain
	local_port  = local_port_base + 1
	remote_port = default_port
	enabled = true

	def extras {
		max_latency = 8.5 # [ms]
	}
}
`)

type Tunnel struct {
	Name       string
	Host       string
	LocalPort  int
	RemotePort int
	Enabled    bool
	Extras     struct {
		MaxLatency float64
	}
}

func main() {
	_ = os.Setenv("VERSION", "1.0.0")
	cfg = bcl.ReplacePlaceholders(cfg)
	var config []Tunnel
	err := bcl.Unmarshal(cfg, &config)
	if err != nil {
		panic(err)
	}
	fmt.Println(strings.ReplaceAll(fmt.Sprintf("%+v", config), " ", "\n  "))

}

My take on including external bcl file

package bcl

import (
	"bufio"
	"bytes"
	"io"
	"os"
	"path/filepath"
	"strings"
)

// ProcessIncludesReader returns an io.Reader that processes @include directives.
// It scans the input line by line, and when encountering a directive like:
//
//	@include "file.bcl"
//
// it replaces that line with the included file's contents (followed by a newline).
// Uses io.MultiReader for minimal allocation.
func ProcessIncludesReader(r io.Reader, basePath string) io.Reader {
	scanner := bufio.NewScanner(r)
	var readers []io.Reader
	for scanner.Scan() {
		line := scanner.Text()
		trimmed := strings.TrimSpace(line)
		if strings.HasPrefix(trimmed, "@include") {
			parts := strings.SplitN(trimmed, " ", 2)
			if len(parts) == 2 {
				includePath := strings.Trim(parts[1], `"`)
				fullPath := filepath.Join(basePath, includePath)
				f, err := os.Open(fullPath)
				if err == nil {
					readers = append(readers, f, bytes.NewReader([]byte("\n")))
					continue
				}
				// If file open fails, insert a comment line as warning.
				readers = append(readers, bytes.NewReader([]byte("// failed to include "+includePath+"\n")))
				continue
			}
		}
		// Regular line followed by a newline.
		readers = append(readers, bytes.NewReader([]byte(line+"\n")))
	}
	return io.MultiReader(readers...)
}

Usage:

// Parse parses the input data, producing executable Prog.
func Parse(input []byte, name string, opts ...Option) (*Prog, error) {
	// Process @include directives.
	r := ProcessIncludesReader(bytes.NewReader(input), ".")
	newInput, err := io.ReadAll(r)
	if err != nil {
		return nil, err
	}
	c := make(chan string, 1)
	c <- string(newInput)
	close(c)
	return parseWithOpts(c, name, opts)
}

// ParseFile reads and parses the input from a BCL file,
// producing executable Prog.
// The input file will be closed as soon as possible.
func ParseFile(f FileInput, opts ...Option) (prog *Prog, _ error) {
	basePath := filepath.Dir(f.Name())
	// Wrap the FileInput with include processing.
	r := ProcessIncludesReader(f, basePath)
	inpc := make(chan string)
	rerr := make(chan error)
	perr := make(chan error)
	done := make(chan struct{})

	go func() {
		defer f.Close()
		var b [4096]byte
		for {
			n, err := r.Read(b[:])
			if err != nil && err != io.EOF {
				rerr <- err
				break
			}
			if err == io.EOF && n == 0 {
				rerr <- nil
				break
			}
			select {
			case inpc <- string(b[:n]):
				continue
			case <-done:
				rerr <- nil
				return
			}
		}
		close(inpc)
	}()

	go func() {
		p, err := parseWithOpts(inpc, f.Name(), opts)
		if err != nil {
			close(done)
		}
		prog = p
		perr <- err
	}()

	err, err2 := <-rerr, <-perr
	if err == nil {
		err = err2
	}
	return prog, err
}

[wkhere]

I like how you did that with a special reader, it's treating the project like a black box and doing minimal changes, it's smart. It seems though it has an disadvantage that the line numbers in error messages are messed up, as they no longer correspond to the original files, but to the joined file fed to the lexer. This is an issue also if one starts to change lexer and parser from inside, so I am going to address that; as mentioned in a comment above, I will embed file names into positions data in the Prog struct. Please stay tuned.

PS. I totally understand that you went for supposting env, because it's a common need. I have that need too. However, I though about implementing getenv as an external function and that, together with a need for functions in general, requires a whole machinery of function call stack. It was stalled for some time but I will get to that; the sketch is in the Robert Nystrom's book.

[wkhere]

Hey, just saying: I had a hiatus with this project but your request is not abandoned:)

[oarkflow]

@wkhere no problem. I've implemented similar project

https://github.com/oarkflow/bcl

[wkhere]

A little note for anyone lurking here: processing multiple files would greatly benefit from implementing functions first.

It's no secret that majority of the parser/vm code borrows from the excellent book "Crafting Interpreters" by Robert Nystrom, modulo the translation from C to Go and some shortcuts taken.
Some of these shortcuts are around having so far simplified chunk/scope/compiler structs and their corresponding code; (also, Chunk struct there is a Prog here); some reorganization of this is needed for multiple files, but also, proper functions need per-function Chunk and embedding multiple Chunks in a Prog; plus corresponding changes in the bytecode serialization.

So, coding this in the opposite order would most probably require trashing a code crafted for multiple inputs as a temporary measure. Better to do it harder but right.