Skip to content

WebAssembly Port #45

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
azizghuloum wants to merge 10 commits into
base: master
Choose a base branch
from
Open

WebAssembly Port #45

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
0803ed1
Added docker file for portable building with no ffmpeg support
azizghuloum Sep 13, 2024
53ac04d
cleaned up Dockerfile
azizghuloum Sep 13, 2024
107c021
WIP web assembly support, FS access not implemented
azizghuloum Sep 14, 2024
9a2c004
taking control of main code and module definition
azizghuloum Sep 14, 2024
f3ffa6c
renamed/added missing main.js file
azizghuloum Sep 14, 2024
db52bd3
most tests running, 3 failing
azizghuloum Sep 15, 2024
0e87521
made a web-suitable compiled file
azizghuloum Sep 16, 2024
807e522
compiled code did not catch exceptions, fixed
azizghuloum Sep 19, 2024
d6acf43
debug build is now optional, with optimized build being the default
azizghuloum Sep 20, 2024
50f109e
added new sections to README
azizghuloum Sep 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
FROM ubuntu:24.04

RUN apt update -y
RUN apt upgrade -y
RUN apt install flex bison g++ make libpng-dev libicu-dev -y

RUN mkdir /context-free
WORKDIR /context-free
COPY Makefile .
COPY src-agg ./src-agg
COPY src-common ./src-common
COPY src-unix ./src-unix
COPY input ./input
RUN make
COPY ./runtests.sh .
RUN make test
39 changes: 34 additions & 5 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@

all: cfdg


#
# Dirs
#
Expand All @@ -20,6 +19,7 @@ vpath %.cfdg input

INC_DIRS = $(COMMON_DIR) $(UNIX_DIR) $(DERIVED_DIR) $(COMMON_DIR)/agg-extras
INC_DIRS += /usr/local/include
#INC_DIRS += /usr/local/include /emsdk/local/include

#
# Installation directories
Expand All @@ -35,6 +35,7 @@ MAN_DIR = $(DESTDIR)$(prefix)/share/man
#

LIB_DIRS = /usr/local/lib
#LIB_DIRS += /emsdk/local/lib

#
# Sources and Objects
Expand Down Expand Up @@ -72,11 +73,17 @@ INPUT_SRCS = ciliasun_v2.cfdg demo1_v2.cfdg demo2_v2.cfdg funky_flower_v2.cfdg \
LIBS = png m

# Use the first one for clang and the second one for gcc
ifeq ($(TARGET), wasm)
LIBS += c++
#LIBS += c++ icui18n icuuc icudata
else
ifeq ($(shell uname -s), Darwin)
LIBS += c++ icucore
else
LIBS += stdc++ atomic icui18n icuuc icudata
endif
endif


#
# FFmpeg support
Expand Down Expand Up @@ -114,6 +121,12 @@ DEPS = $(patsubst %.o,%.d,$(OBJS))
LINKFLAGS += $(patsubst %,-L%,$(LIB_DIRS))
LINKFLAGS += $(patsubst %,-l%,$(LIBS))
LINKFLAGS += -fexceptions
ifeq ($(TARGET), wasm)
ifeq ($(WASM_DEBUG), 1)
LINKFLAGS += -s EXCEPTION_DEBUG=1 -s SYSCALL_DEBUG=1 -s FS_DEBUG=1 -s ASSERTIONS=2 -g -gsource-map
endif
LINKFLAGS += -fexceptions -s SUPPORT_LONGJMP=emscripten -s USE_LIBPNG=1 -s USE_ICU=1 -s INVOKE_RUN=0 -s EXPORTED_RUNTIME_METHODS=callMain,FS -s ALLOW_MEMORY_GROWTH=1 -s STACK_SIZE=104857600
endif

deps: $(OBJ_DIR) $(DEPS)

Expand All @@ -132,9 +145,15 @@ $(OBJS): $(OBJ_DIR)/Sentry
#
# Under Cygwin replace strip $@ with strip $@.exe

ifeq ($(TARGET), wasm)
STRIP = echo
else
STRIP = strip
endif

cfdg: $(OBJS)
$(LINK.o) $^ $(LINKFLAGS) -o $@
strip $@
$(STRIP) $@


#
Expand Down Expand Up @@ -184,10 +203,13 @@ uninstall:
# Tests
#

.PHONY: test check
.PHONY: test test-mpeg check
test: cfdg
./runtests.sh

test-mpeg: cfdg
./runtests-mpeg.sh

check: cfdg
./runtests.sh

Expand All @@ -197,8 +219,8 @@ check: cfdg

CXXFLAGS += $(patsubst %,-I%,$(INC_DIRS))
CXXFLAGS += -O2 -Wall -Wextra -Wno-parentheses -std=c++17
CXXFLAGS += -g -D_GLIBCXX_USE_C99_MATH=1
CPPFLAGS += -DNDEBUG
CXXFLAGS += -D_GLIBCXX_USE_C99_MATH=1
#CPPFLAGS += -DNDEBUG

# Add this for clang
ifeq ($(shell uname -s), Darwin)
Expand All @@ -208,6 +230,13 @@ ifeq ($(shell uname -s), Darwin)
# LDFLAGS += -framework CoreFoundation -framework CoreVideo -framework CoreMedia -framework VideoToolbox
endif

ifeq ($(TARGET), wasm)
CXXFLAGS += -DNOSYSCTL=1 -DNONORMALIZE=1 -fexceptions -s SUPPORT_LONGJMP=emscripten
ifeq ($(WASM_DEBUG), 1)
CXXFLAGS += -g -gsource-map
endif
endif

$(OBJ_DIR)/%.o : %.cpp
$(COMPILE.cpp) $(OUTPUT_OPTION) $<

Expand Down
35 changes: 35 additions & 0 deletions README
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,3 +127,38 @@ will install cfdg and the cfdg.1 man page in /usr/local.
FFmpeg BUILD NOTES

Check out README.ffmpeg for building ffmpeg and enabling ffmpeg support

~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~
DOCKER BUILD NOTES

You can build and run the Linux/Posix version of cfdg inside a docker container
by running the command:

$ docker build . -f Dockerfile

This will install the operating system and all the build tools (GCC, Make, etc)
inside an image, copy the cfdg files needed for the build, compile it, then run
the tests. Note that to use cfdg inside a container, you need to mount your
files/folders into the container then execute the cfdg command on the mounted
files.

~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~
WEBASSEMBLY

You can compile cfdg to wasm by running the command:

$ docker build . -f Wasm.Dockerfile

This will install the operating system and all the build tools (Emscripten and
friends), then build and test cfdg wasm scripts/libraries.

At the end, the docker file will produce two artifacts:

(1) The browser-suitable library:
/context-free/web/cfdg.js

(2) NodeJS executable script:
/context-free/cfdg

The executable script is used for running the tests. The web library is not
tested here.
39 changes: 39 additions & 0 deletions Wasm.Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
FROM ubuntu:24.04

RUN apt update -y
RUN apt upgrade -y
RUN apt install git -y
RUN git clone --depth=1 https://github.com/emscripten-core/emsdk.git
WORKDIR /emsdk
RUN apt install python3 -y
RUN apt install xz-utils -y
RUN ./emsdk install latest
RUN ./emsdk activate latest
ENV EMSDK=/emsdk
ENV PATH="/emsdk:/emsdk/upstream/emscripten:/emsdk/node/18.20.3_64bit/bin:${PATH}"
RUN echo 'int main(){return 0;}' > hello.cpp && em++ -s USE_LIBPNG=1 -s USE_ICU=1 hello.cpp && rm a.out.wasm a.out.js

RUN apt install flex bison make g++ vim -y

RUN cp /usr/include/FlexLexer.h /usr/local/include

RUN mkdir /context-free
WORKDIR /context-free
COPY Makefile .
COPY src-agg ./src-agg
COPY src-common ./src-common
COPY src-unix ./src-unix
RUN mkdir ./input
COPY ./input/*.cfdg ./input

RUN TARGET=wasm LINKFLAGS='-s ENVIRONMENT=web,worker -s WASM=1 -s SINGLE_FILE=1' emmake make
RUN mkdir web && mv cfdg cfdg.js && mv cfdg.* web

RUN TARGET=wasm LINKFLAGS='-lnodefs.js -lnoderawfs.js' emmake make
RUN mv cfdg cfdg.js
COPY src-js/main.js cfdg
RUN touch cfdg # avoid make recompiling stuff
COPY ./input/*.SKIP ./input
COPY ./input/tests ./input/tests
COPY ./runtests.sh .
RUN make test
3 changes: 3 additions & 0 deletions runtests-mpeg.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
#!/bin/sh

./cfdg -a150 -s640x480 --quicktime -vffgh input/mtree.cfdg output/mtree.mpeg
4 changes: 1 addition & 3 deletions runtests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
#!/bin/sh

mkdir output
for file in input/tests/*.cfdg input/*.cfdg
do
./cfdg -qP "$file" output/test.png
./cfdg -P "$file" output/test.png
if [ $? -eq 0 ]
then
echo "$file pass"
Expand All @@ -12,5 +11,4 @@ do
break
fi
done
./cfdg -a150 -s640x480 --quicktime -vffgh input/mtree.cfdg output/mtree.mpeg

15 changes: 15 additions & 0 deletions src-js/main.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
#!/usr/bin/env node

var cfdg = require('./cfdg.js');
var fs = require('fs');

cfdg.onRuntimeInitialized = () => {
// console.log(cfdg);
// console.log(cfdg.FS);
// console.log(process.cwd());
// console.log(cfdg.FS.cwd());
// console.log(cfdg.FS.readdir(cfdg.FS.cwd()));
var [_node, _script, ...args] = [...process.argv];
cfdg.callMain(args);
};

31 changes: 24 additions & 7 deletions src-unix/posixSystem.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@
#define UCHAR_TYPE char16_t
#include <unicode/ucnv.h>
#include <unicode/unorm2.h>
#include <unicode/putil.h>
#include <unicode/unistr.h> // ICU header for UnicodeString


#include <cstdlib>
#include <iostream>
Expand All @@ -45,13 +48,15 @@
#include <dirent.h>
#include <cstring>

#ifndef NOSYSCTL
#if defined(__GNU__) || (defined(__ILP32__) && defined(__x86_64__))
#define NOSYSCTL
#else
#ifndef __linux__
#include <sys/sysctl.h>
#endif
#endif
#endif
#include <sys/types.h>
#include <unistd.h>
#include <cstdint>
Expand Down Expand Up @@ -219,20 +224,32 @@ PosixSystem::~PosixSystem()
ucnv_close(mConverter);
}

#ifdef NONORMALIZE
std::wstring
PosixSystem::normalize(const std::string& u8name) {
icu::UnicodeString ustr = icu::UnicodeString::fromUTF8(icu::StringPiece(u8name.c_str()));
std::wstring wstr(ustr.length(), L' ');
for (int32_t i = 0; i < ustr.length(); ++i) {
wstr[i] = ustr.charAt(i);
}
return wstr;
}
#else
std::wstring
PosixSystem::normalize(const std::string& u8name)
{
// Setup ICU library items
UErrorCode status = U_ZERO_ERROR;
if (!mConverter)
mConverter = ucnv_open("utf-8", &status);
if (!mNormalizer && U_SUCCESS(status))
if (U_FAILURE(status)) {
catastrophicError("No Converter");
}
if (!mNormalizer)
mNormalizer = unorm2_getNFKCInstance(&status);
if (!mConverter || !mNormalizer) {
if (!mErrorReported)
catastrophicError("String conversion initialization error");
mErrorReported = true;
return std::wstring();
if (U_FAILURE(status)) {
std::cerr << "Error getting NFKC normalizer: " << u_errorName(status) << std::endl;
catastrophicError("No Normalizer");
}

// Convert from utf-8 to utf-16
Expand Down Expand Up @@ -273,4 +290,4 @@ PosixSystem::normalize(const std::string& u8name)
}
return ret;
}

#endif