From bbc049b5c7749b8f9c4e1ade9cf1da4f91f37c0d Mon Sep 17 00:00:00 2001
From: Camil Staps
Date: Wed, 23 Jun 2021 19:20:29 +0200
Subject: Add article series on the clean sandbox
---
gulpfile.js | 15 +
package-lock.json | 48 ++-
package.json | 5 +-
...browser-with-webassembly-part-1-introduction.md | 83 ++++++
...browser-with-webassembly-part-2-the-pipeline.md | 328 +++++++++++++++++++++
...h-webassembly-part-3-putting-it-all-together.md | 193 ++++++++++++
...ean-compiler-in-the-browser-with-webassembly.md | 7 -
...rowser-with-webassembly-part-1-introduction.pug | 18 ++
...rowser-with-webassembly-part-2-the-pipeline.pug | 18 ++
...-webassembly-part-3-putting-it-all-together.pug | 18 ++
...an-compiler-in-the-browser-with-webassembly.pug | 12 -
resources/pug/finals/articles/index.pug | 16 +-
resources/pug/finals/ham/cw-decoder.pug | 6 +-
resources/pug/include/layout-articles.pug | 13 +-
resources/pug/include/layout-ham.pug | 3 +
resources/pug/include/layout-sidebar.pug | 44 ++-
resources/pug/include/layout.pug | 1 -
resources/sass/style.scss | 20 ++
18 files changed, 803 insertions(+), 45 deletions(-)
create mode 100644 resources/md/2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-1-introduction.md
create mode 100644 resources/md/2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-2-the-pipeline.md
create mode 100644 resources/md/2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-3-putting-it-all-together.md
delete mode 100644 resources/md/2021-06-23-running-the-clean-compiler-in-the-browser-with-webassembly.md
create mode 100644 resources/pug/finals/articles/2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-1-introduction.pug
create mode 100644 resources/pug/finals/articles/2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-2-the-pipeline.pug
create mode 100644 resources/pug/finals/articles/2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-3-putting-it-all-together.pug
delete mode 100644 resources/pug/finals/articles/2021-06-23-running-the-clean-compiler-in-the-browser-with-webassembly.pug
diff --git a/gulpfile.js b/gulpfile.js
index 37a07c5..8fd7481 100644
--- a/gulpfile.js
+++ b/gulpfile.js
@@ -3,6 +3,9 @@ var
gulp = require('gulp'),
hljs = require('highlight.js'),
md = require('jstransformer')(require('jstransformer-markdown-it')),
+ md_anchor = require('markdown-it-anchor'),
+ md_footnote = require('markdown-it-footnote'),
+ md_toc = require('markdown-it-table-of-contents'),
minifyhtml = require('gulp-minify-html'),
minifyjs = require('gulp-minify'),
notify = require('gulp-notify'),
@@ -56,8 +59,20 @@ function html() {
return '';
},
+ html: true,
langPrefix: 'lang-',
linkify: true,
+ plugins: [
+ [md_anchor, {
+ permalink: md_anchor.permalink.ariaHidden({
+ placement: 'before'
+ }),
+ }],
+ md_footnote,
+ [md_toc, {
+ containerHeaderHtml: 'Contents
',
+ }],
+ ],
}, options);
return md.render(text, options).body;
}
diff --git a/package-lock.json b/package-lock.json
index 8ce8b8b..fb73791 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -21,7 +21,10 @@
"gulp-sass-next": "^6.0.0",
"highlight.js": "^11.0.1",
"jquery": "^3.6.0",
- "jstransformer-markdown-it": "^2.1.0"
+ "jstransformer-markdown-it": "^2.1.0",
+ "markdown-it-anchor": "^8.0.3",
+ "markdown-it-footnote": "^3.0.3",
+ "markdown-it-table-of-contents": "^0.5.2"
}
},
"node_modules/@babel/helper-validator-identifier": {
@@ -4470,6 +4473,30 @@
"markdown-it": "bin/markdown-it.js"
}
},
+ "node_modules/markdown-it-anchor": {
+ "version": "8.0.3",
+ "resolved": "https://registry.npmjs.org/markdown-it-anchor/-/markdown-it-anchor-8.0.3.tgz",
+ "integrity": "sha512-YU5kfZDKjqRiY6wySd3ouk6wVmByUczPCeTvKWCl7St/AnYEkiCxj29T1pK0phuvJFZJc1aNVoSFqzZ7I4XB/A==",
+ "dev": true,
+ "peerDependencies": {
+ "markdown-it": "*"
+ }
+ },
+ "node_modules/markdown-it-footnote": {
+ "version": "3.0.3",
+ "resolved": "https://registry.npmjs.org/markdown-it-footnote/-/markdown-it-footnote-3.0.3.tgz",
+ "integrity": "sha512-YZMSuCGVZAjzKMn+xqIco9d1cLGxbELHZ9do/TSYVzraooV8ypsppKNmUJ0fVH5ljkCInQAtFpm8Rb3eXSrt5w==",
+ "dev": true
+ },
+ "node_modules/markdown-it-table-of-contents": {
+ "version": "0.5.2",
+ "resolved": "https://registry.npmjs.org/markdown-it-table-of-contents/-/markdown-it-table-of-contents-0.5.2.tgz",
+ "integrity": "sha512-6o+rxSwzXmXCUn1n8QGTSpgbcnHBG6lUU8x7A5Cssuq5vbfzTfitfGPvQ5PZkp+gP1NGS/DR2rkYqJPn0rbZ1A==",
+ "dev": true,
+ "engines": {
+ "node": ">6.4.0"
+ }
+ },
"node_modules/matchdep": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/matchdep/-/matchdep-2.0.0.tgz",
@@ -11256,6 +11283,25 @@
"uc.micro": "^1.0.5"
}
},
+ "markdown-it-anchor": {
+ "version": "8.0.3",
+ "resolved": "https://registry.npmjs.org/markdown-it-anchor/-/markdown-it-anchor-8.0.3.tgz",
+ "integrity": "sha512-YU5kfZDKjqRiY6wySd3ouk6wVmByUczPCeTvKWCl7St/AnYEkiCxj29T1pK0phuvJFZJc1aNVoSFqzZ7I4XB/A==",
+ "dev": true,
+ "requires": {}
+ },
+ "markdown-it-footnote": {
+ "version": "3.0.3",
+ "resolved": "https://registry.npmjs.org/markdown-it-footnote/-/markdown-it-footnote-3.0.3.tgz",
+ "integrity": "sha512-YZMSuCGVZAjzKMn+xqIco9d1cLGxbELHZ9do/TSYVzraooV8ypsppKNmUJ0fVH5ljkCInQAtFpm8Rb3eXSrt5w==",
+ "dev": true
+ },
+ "markdown-it-table-of-contents": {
+ "version": "0.5.2",
+ "resolved": "https://registry.npmjs.org/markdown-it-table-of-contents/-/markdown-it-table-of-contents-0.5.2.tgz",
+ "integrity": "sha512-6o+rxSwzXmXCUn1n8QGTSpgbcnHBG6lUU8x7A5Cssuq5vbfzTfitfGPvQ5PZkp+gP1NGS/DR2rkYqJPn0rbZ1A==",
+ "dev": true
+ },
"matchdep": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/matchdep/-/matchdep-2.0.0.tgz",
diff --git a/package.json b/package.json
index 3a80179..4103d0a 100644
--- a/package.json
+++ b/package.json
@@ -12,7 +12,10 @@
"gulp-sass-next": "^6.0.0",
"highlight.js": "^11.0.1",
"jquery": "^3.6.0",
- "jstransformer-markdown-it": "^2.1.0"
+ "jstransformer-markdown-it": "^2.1.0",
+ "markdown-it-anchor": "^8.0.3",
+ "markdown-it-footnote": "^3.0.3",
+ "markdown-it-table-of-contents": "^0.5.2"
},
"dependencies": {
"@fortawesome/fontawesome-free": "^5.15.3",
diff --git a/resources/md/2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-1-introduction.md b/resources/md/2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-1-introduction.md
new file mode 100644
index 0000000..b118b28
--- /dev/null
+++ b/resources/md/2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-1-introduction.md
@@ -0,0 +1,83 @@
+*My [interpreter][abc-interpreter] for the pure, functional language [Clean][]
+allows interpretation in both C and WebAssembly. The Clean compiler is written
+in Clean, with a C backend. In my [Clean Sandbox][], I compile the C backend
+with [Emscripten][] to [WebAssembly][], and interpret the Clean frontend in the
+WebAssembly interpreter. Combined with a simple `make`-like tool and an editor,
+this allows you to compile and run Clean code in the browser.*
+
+[[toc]]
+
+# Introduction
+
+Browsers can run programs in only few languages; mainly JavaScript. But
+JavaScript is a high-level scripting language that makes it hard to optimize
+performance. [WebAssembly][] operates on a lower level than JavaScript, is more
+complicated to write code in, but is a relatively easy compilation target for
+common imperative languages like C. This means that C applications and
+libraries can be ported to the browser relatively easily.
+
+However, WebAssembly lacks some features that make it a difficult compilation
+target for functional programming languages (in particular, non-local control
+flow and tail calls). Some of these omissions are being implemented as post-MVP
+features, but until then we need a workaround.
+
+Back in 2019 I implemented such a workaround for the pure, functional
+programming language [Clean][] (a close cousin of Haskell). The Clean compiler
+compiles to the intermediate language ABC, which is designed for an abstract
+imperative machine for graph rewriting. This ABC code is then converted to
+actual machine code. My solution, published in IFL'2019,[^ifl-2019] involves an
+interpreter for this intermediate ABC language. The interpreter logic is
+written in a DSL which targets both C and WebAssembly, so that you can run it
+both natively and in the browser.
+
+Initially, we used this system in the [iTasks][] framework for workflow
+management web applications. There, it allows you to send arbitrary Clean
+values (including thunks) to the browser for evaluation there. This has two
+benefits: (1) reducing server load by evaluating more on the client; (2) the
+ability to write custom frontend components in Clean, which allows you to make
+use of the same utility functions, the benefits of strong typing, etc.
+
+However, the ABC interpreter is more widely applicable than the iTasks system.
+To demonstrate this, I set out to run the Clean compiler itself in the browser.
+You can see the result in the [Clean Sandbox][].
+
+# High-level overview
+
+In the end, the architecture turned out to be relatively simple. I needed the
+following components:
+
+- A file system to keep the files (taken from the Emscripten library).
+- A simple `make`-like tool that checks which files have been modified and
+ calls the compiler for those files only.
+- The compiler frontend (written in Clean), running in the ABC interpreter.
+- The compiler backend (written in C), compiled with Emscripten to
+ WebAssembly.
+- A linker for the generated ABC files (written in C, compiled to WebAssembly).
+- The ABC interpreter to run the resulting 'executable'.
+- A nice editor to let the user modify his Clean code. I use something based on
+ the [Monaco Editor][], but will not discuss it here as I want to focus on the
+ build pipeline.
+
+Apart from the `make`-like tool, all these components were already available, I
+only had to connect them together.
+
+*Continue to read about the [implementation details][pipeline].*
+
+[^ifl-2019]:
+ Camil Staps, John van Groningen and Rinus Plasmeijer, 2019.
+ [Lazy Interworking of Compiled and Interpreted Code for Sandboxing and Distributed Systems](https://camilstaps.nl/assets/pdf/ifl2019.pdf),
+ in Jurriën Stutterheim and Wei Ngan Chin (eds.),
+ *Implementation and Application of Functional Languages (IFL '19), September 25–27, 2019, Singapore, Singapore*.
+ doi: [10.1145/3412932.3412941](https://doi.org/10.1145/3412932.3412941)
+
+[abc-interpreter]: https://gitlab.com/clean-and-itasks/abc-interpreter/
+[Clean]: http://clean.cs.ru.nl/
+[Clean Sandbox]: https://camilstaps.gitlab.io/clean-sandbox/
+[Emscripten]: https://emscripten.org/
+[iTasks]: https://gitlab.com/clean-and-itasks/itasks-sdk/
+[Monaco Editor]: https://microsoft.github.io/monaco-editor/
+[WebAssembly]: https://webassembly.org/
+
+[introduction]: 2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-1-introduction.html
+[pipeline]: 2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-2-the-pipeline.html
+[integration]: 2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-3-putting-it-all-together.html
diff --git a/resources/md/2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-2-the-pipeline.md b/resources/md/2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-2-the-pipeline.md
new file mode 100644
index 0000000..b766560
--- /dev/null
+++ b/resources/md/2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-2-the-pipeline.md
@@ -0,0 +1,328 @@
+*This is part 2 in a series on running the [Clean][] compiler in
+[WebAssembly][], with the proof of concept in the [Clean Sandbox][]. In this
+part I discuss the compilation pipeline and the program used to rebuild
+generated files. See the [introduction][] for a high-level overview.*
+
+[[toc]]
+
+# The compilation pipeline
+
+Basically, this is what we do when we compile a Clean program:
+
+1. For each Clean file, generate intermediate ABC code.
+2. For each ABC file, generate machine code.
+3. Link the machine code together with the system linker into an executable.
+
+When we compile for interpretation, this looks a little different:
+
+1. For each Clean file, generate intermediate ABC code.
+2. For each ABC file, generate ABC bytecode.
+
+ Normal ABC code is human-readable. ABC bytecode is easier to parse.
+ Furthermore, the ABC bytecode defines some custom instructions to speed up
+ interpretation, like frequent instruction combinations (e.g. a `pop`
+ followed by a `return`).
+3. Link the ABC bytecode together with the ABC linker.
+4. Prelink the ABC bytecode for use in the WebAssembly interpreter.
+
+ Prelinking removes symbol names and relocations, assuming the code segment
+ starts at address 0. This is possible because the WebAssembly memory always
+ starts at address 0. This way, the WebAssembly interpreter does not need to
+ deal with relocations.
+
+For step 1, we use the Clean compiler, written in Clean and C. The other tools
+are maintained in the [abc-interpreter][] repository, and are written in C. We
+compile the compiler frontend to prelinked bytecode, and the C tools to
+WebAssembly using emscripten.
+
+# The file system
+
+Because the Clean compilation pipeline expects to read and write files (rather
+than `stdin` and `stdout`), we need to set up a file system. Luckily,
+[Emscripten][] already includes one with the
+[`FS` library](https://emscripten.org/docs/api_reference/Filesystem-API.html).
+I use the `FS` library for file access, and the `IDBFS` backend, which is based
+on IndexedDB, for persistent storage.
+
+Unfortunately, it is [currently not possible to let the different C tools (code
+generator, linker, etc.) share the same file
+system](https://groups.google.com/g/emscripten-discuss/c/_K61fo-9oKY/m/4m6LYrcPo5sJ).
+To circumvent this problem, I link the file system of each tool to the same
+IndexedDB. When switching from one tool to another (e.g., going from the code
+generator to the linker), I sync the file systems with the IndexedDB. This
+incurs some overhead, but makes sure that the contents are the same everywhere.
+
+There is a `make`-like tool which checks which files need to be rebuilt based
+on timestamps. This tool is written in Clean, compiled to prelinked bytecode,
+and runs in the WebAssembly interpreter.
+
+# Building the C tools
+
+Each C tool is built to its own JavaScript and WebAssembly source using
+[Emscripten][]. This is similar to building these tools to a native executable,
+except that we use `emcc` instead of `gcc` and add some other options. For
+example, the bytecode generator is normally built like this:
+
+```make
+CLIBS:=-lm
+
+BCGEN:=bcgen
+
+SRC_BCGEN:=abc_instructions.c bcgen.c bcgen_instructions.c bcgen_instruction_table.c bytecode.c parse_abc.c util.c
+DEP_BCGEN:=$(subst .c,.h,$(SRC_BCGEN)) settings.h
+
+$(BCGEN): $(SRC_BCGEN) $(DEP_BCGEN)
+ $(CC) $(CFLAGS) $(SRC_BCGEN) $(CLIBS) -DBCGEN -o $@
+```
+
+With Emscripten we use:
+
+```make
+EMCC:=emcc
+
+JS_BCGEN:=bcgen.js
+
+$(JS_BCGEN): $(SRC_BCGEN)
+ $(EMCC) -O2 -DPOSIX -DBCGEN\
+ -s ENVIRONMENT=web\
+ -s INVOKE_RUN=0\
+ -s MODULARIZE -s EXPORT_NAME=bcgen\
+ -lidbfs.js -s FORCE_FILESYSTEM=1 -s EXPORTED_RUNTIME_METHODS=[FS,IDBFS]\
+ $^ -o $@
+```
+
+This compiles the C sources to WebAssembly and generates a file `bcgen.js` that
+acts as a wrapper around it. Loading `bcgen.js` will then define the function
+`bcgen` which returns a promise resolving in an Emscripten module with the
+property `_main`, corresponding to the C function `main`.
+
+This sounds complicated, but we can instantiate the Emscripten modules like
+this:
+
+```html
+
+
+
+```
+
+```js
+var c_bcgen, c_bclink, c_bcprelink;
+Promise.all([
+ bcgen().then(instance => c_bcgen = instance),
+ bclink().then(instance => c_bclink = instance),
+ bcprelink().then(instance => c_bcprelink = instance),
+])
+```
+
+# Building the compiler
+
+The Clean frontend of the compiler is relatively easy to build. We take the
+normal project file (Clean's `Makefile`) and use `cpm` (Clean's `make`) to
+build the prelinked bytecode.
+
+The C backend is a little trickier. We use the same `make` recipe as above.
+However, the compiler expects some global functions to be defined. These
+functions are normally provided by the native Clean run-time system or a
+supporting C library, but are not present in the interpreter. We need to
+define:
+
+- `set_return_code` which takes an `int` and sets the process exit code.
+- `ArgEnvCopyCStringToCleanStringC` which takes a null-terminated C string and
+ builds a Clean string from it (which is not null-terminated but includes the
+ number of bytes).
+- `ArgEnvGetCommandLineArgumentC` which takes an `int` and returns a pointer to
+ a C string with the *n*th command-line argument.
+- `ArgEnvGetCommandLineCountC` which returns the number of command-line
+ arguments (`argc`).
+
+To implement these functions, we make use of custom properties on the
+Emscripten module: `clean_argc`, `clean_argv`, and `clean_return_code`. The
+JavaScript looks like this:
+
+```js
+var c_compiler;
+compiler().then(instance => {
+ c_compiler = instance;
+
+ c_compiler._ArgEnvGetCommandLineCountC = () => c_compiler.clean_argc;
+
+ c_compiler._ArgEnvGetCommandLineArgumentC = (i, sizep, sp) => {
+ const arg = c_compiler.HEAP32[c_compiler.clean_argv/4+i];
+ var p = arg;
+ for (; c_compiler.HEAPU8[p] != 0; p++);
+ c_compiler.HEAP32[sizep/4] = p-arg;
+ c_compiler.HEAP32[sp/4] = arg;
+ };
+
+ c_compiler._ArgEnvCopyCStringToCleanStringC = (cs, cleans) => {
+ const size = c_compiler.HEAP32[cleans/4];
+ cleans += 4;
+ for (var i = 0; i < size; i++)
+ c_compiler.HEAPU8[cleans++] = c_compiler.HEAPU8[cs++];
+ };
+ c_compiler._ArgEnvCopyCStringToCleanStringC.sync_strings_back_to_clean = true;
+
+ c_compiler._set_return_code = (c) => {c_compiler.clean_return_code = c;};
+});
+```
+
+# Preparing the file system
+
+With all the tools set up, we need to create the main directories in the file
+system. We do this by adding them in the file system of the `c_compiler`, and
+then sync the other file systems.
+
+```js
+c_compiler.FS.mkdir('/clean');
+c_compiler.FS.mount(c_compiler.IDBFS, {}, '/clean');
+c_compiler.FS.mkdir('/clean/lib');
+c_compiler.FS.mkdir('/clean/src');
+
+const mount = backend => new Promise(resolve => {
+ backend.FS.mkdir('/clean');
+ backend.FS.mount(backend.IDBFS, {}, '/clean');
+ backend.FS.syncfs(true, err => {
+ if (err)
+ throw err;
+ backend.FS.chdir('/clean/src');
+ resolve();
+ });
+});
+
+return new Promise(resolve => {
+ c_compiler.FS.syncfs(false, () => {
+ Promise.all([
+ mount(c_bcgen),
+ mount(c_bclink),
+ mount(c_bcprelink),
+ ]).then(resolve);
+ });
+});
+```
+
+In the real sandbox, we also add some standard libraries to the file system on
+start-up.
+
+# The `make`-like tool
+
+I had to build a simple `make`-like tool to call the compiler and regenerate
+files when needed. This tool is a separate Clean program, which is compiled to
+prelinked bytecode and interpreted in WebAssembly.
+
+The following snippet sets everything up: it sets the global variable
+`sandbox.compile` to a Clean function that makes the sandbox project. In the
+[last part][integration] I show how everything is put together, here I want to
+focus on the implementation.
+
+`wrapInitFunction` makes this a Clean program that can run in the browser. The
+`me` parameter can be ignored here. The `compile` function calls `make`, which
+contains the actual logic. `make` is a higher-order function which takes
+functions for compilation, code generation, etc., as parameters. These are here
+provided as `call_main c_bcgen` etc.
+
+```clean
+Start = wrapInitFunction start
+
+sandbox :== jsGlobal "sandbox"
+c_compiler :== jsGlobal "c_compiler"
+clean_compiler :== jsGlobal "clean_compiler"
+c_bcgen :== jsGlobal "c_bcgen"
+c_bclink :== jsGlobal "c_bclink"
+c_bcprelink :== jsGlobal "c_bcprelink"
+
+start :: !JSVal !*JSWorld -> *JSWorld
+start me w
+ # (cb,w) = jsWrapFun (compile me) me w
+ # w = (sandbox .# "compile" .= cb) w
+ = w
+where
+ compile me {[0]=paths,[1]=main,[2]=callback} w
+ # (paths,w) = jsValToList` paths (fromJS "") w
+ = make
+ /* .. */
+ do_compile
+ (call_main c_bcgen) (call_main c_bclink) (call_main c_bcprelink)
+ /* .. */
+ paths (fromJS "" main)
+ w
+```
+
+The C tools we can call relatively easily, by copying `argv` into the memory of
+the process, setting `argc`, and calling `main`. Afterwards we need to free the
+memory:
+
+```clean
+ call_main backend args w
+ # (argc,args,argv,w) = copy_argv args backend w
+ # (res,w) = (backend .# "_main" .$? (argc, argv)) (-1, w)
+ = (res == 0, free_args_and_argv args argv backend w)
+```
+
+Compilation is a little trickier, because it is a Clean program. For this the
+`clean_compiler` object, which is a WebAssembly interpreter, has a `.compile()`
+method. This method makes sure that no global state is kept between runs (by
+clearing CAFs), and that the memory does not get corrupted if compilation
+fails:
+
+```js
+clean_compiler.compile = function () {
+ clean_compiler.clear_cafs();
+ c_compiler.clean_stdio_open = false;
+
+ var res = 0;
+ try {
+ /* Interpret from the main entry point */
+ clean_compiler.interpreter.instance.exports.set_pc(clean_compiler.start);
+ clean_compiler.interpreter.instance.exports.interpret();
+ res = c_compiler.clean_return_code;
+ } catch (e) {
+ console.warn(e);
+ res = e;
+ /* Memory may be badly corrupted, create a new instance to be sure */
+ create_clean_compiler();
+ }
+
+ /* Clear output buffers */
+ if (sandbox.stdout_buffer.length > 0)
+ sandbox.stdout('\n');
+ if (sandbox.stderr_buffer.length > 0)
+ sandbox.stderr('\n');
+
+ return res;
+};
+```
+
+The Clean wrapper also checks the exit code of the process, and catches any
+errors:
+
+```clean
+ do_compile args w
+ # (_,args,argv,w) = copy_argv args c_compiler w
+ # (res_or_err,w) = (clean_compiler .# "compile" .$ ()) w
+ # w = free_args_and_argv args argv c_compiler w
+ # res = jsValToInt res_or_err
+ | isJust res
+ = (fromJust res==0, w)
+ # (err,w) = (res_or_err .# "toString" .$? ()) ("Compiler crashed.", w)
+ = (False, feedback False err w)
+```
+
+The make tool itself is now relatively simple. It keeps a queue of modules to
+be compiled. It takes a module from the queue, compiles it, and checks the
+generated code for any dependencies. These dependencies are added to the queue,
+if they were not compiled yet. When the queue is emptied, the same is done for
+bytecode generation. Finally, the generated code is linked into a single
+bytecode file and prelinked bytecode file.
+
+*That's all about the pipeline. You can continue reading about [how it's all
+put together][integration].*
+
+[abc-interpreter]: https://gitlab.com/clean-and-itasks/abc-interpreter/
+[Clean]: http://clean.cs.ru.nl/
+[Clean Sandbox]: https://camilstaps.gitlab.io/clean-sandbox/
+[Emscripten]: https://emscripten.org/
+[WebAssembly]: https://webassembly.org/
+
+[introduction]: 2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-1-introduction.html
+[pipeline]: 2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-2-the-pipeline.html
+[integration]: 2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-3-putting-it-all-together.html
diff --git a/resources/md/2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-3-putting-it-all-together.md b/resources/md/2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-3-putting-it-all-together.md
new file mode 100644
index 0000000..0244507
--- /dev/null
+++ b/resources/md/2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-3-putting-it-all-together.md
@@ -0,0 +1,193 @@
+*This is part 3 in a series on running the [Clean][] compiler in
+[WebAssembly][], with the proof of concept in the [Clean Sandbox][]. In this
+part I discuss how all the individual components are integrated. See the
+[introduction][] for a high-level overview. In the previous part, I discussed
+[the compilation pipeline][pipeline].*
+
+[[toc]]
+
+# Overview
+
+We have to put all our different elements together in JavaScript. In particular
+we have the following elements:
+
+- The C tools (compiler backend, bytecode generator, bytecode linker, and
+ bytecode prelinker)
+- The WebAssembly interpreters in which the Clean tools run (compiler frontend,
+ `make`-like program)
+- The WebAssembly interpreter for the compiled program itself.
+
+# Linking the compiler frontend and backend
+
+We have already seen how [the `make` tool communicates with the C tools and
+with the compiler][pipeline]. It is a little trickier to set up the
+communication between the compiler frontend and the backend. This is because
+the `make` tool was written for this application specifically, while the
+compiler wasn't.
+
+In Clean, the foreign function interface works through a special ABC
+instruction, `ccall`. For example, if we have a C function `int c_add(int,
+int)`, we can define `add` below. `II:I` here stands for the type: two integer
+arguments and an integer return type.
+
+```clean
+add :: !Int !Int -> Int
+add _ _ = code {
+ ccall c_add "II:I"
+}
+```
+
+This instruction is not supported by the interpreter, because there is no way
+to implement it generically in that context. When interpretation reaches a
+`ccall` (or any other unimplemented instruction), the WebAssembly interpreter
+calls a JavaScript function that can try to handle it. This function can read
+and modify the state of the ABC machine, and return to it after it has handled
+the instruction. This allows us to link the compiler frontend to the backend in
+JavaScript:
+
+```js
+var clean_compiler;
+function create_clean_compiler () {
+ return ABCInterpreter.instantiate({
+ interpreter_imports: {
+ handle_illegal_instr: (pc, instr, asp, bsp, csp, hp, hp_free) => {
+ instr = ABCInterpreter.instructions[instr];
+ if (instr == 'ccall')
+ return i_ccall.bind(c_compiler)(clean_compiler, pc, asp, bsp);
+ else
+ return 0;
+ },
+ /* other options .. */
+ }
+ }).then(abc => {
+ clean_compiler = abc;
+ });
+}
+```
+
+With `i_ccall` we can link a WebAssembly interpreter to an Emscripten module,
+which contains the global C functions:
+
+```js
+function i_ccall (abc, pc, asp, bsp) {
+ const fun = '_' + abc.get_clean_string(abc.memory_array[pc/4+2]-8); /* e.g. c_add; emscripten adds an underscore */
+ var type = abc.get_clean_string (abc.memory_array[pc/4+4]-8); /* e.g. II:I */
+
+ if (!(fun in this))
+ throw 'ccall: unknown function '+fun;
+
+ /* parse type; get arguments from the Clean heap and stack .. */
+
+ const result = this[fun].apply(null, args);
+
+ /* copy the result to the WebAssembly interpreter .. */
+
+ return pc+24; /* return the address of the next instruction */
+}
+```
+
+# Clean file I/O
+
+We have a similar problem with Clean programs doing file I/O. The compiler
+frontend uses file I/O, but for this it needs ABC instructions that are not
+implemented in the WebAssembly interpreter. We extend `handle_illegal_instr` to
+also catch these instructions, and use Emscripten's `FS` library to implement
+them. For example, `writeFS` writes a string to a file. `handle_illegal_instr`
+becomes:
+
+```js
+handle_illegal_instr: (pc, instr, asp, bsp, csp, hp, hp_free) => {
+ instr = ABCInterpreter.instructions[instr];
+ if (instr == 'ccall')
+ return i_ccall.bind(c_compiler)(clean_compiler, pc, asp, bsp);
+ else if (instr == 'writeFS')
+ return i_writeFS.bind(c_compiler)(clean_compiler, pc, asp, bsp);
+ else
+ return 0;
+},
+```
+
+And again `i_witeFS` is defined to link an arbitrary WebAssembly interpreter
+and Emscripten module together:
+
+```js
+
+function i_writeFS (abc, pc, asp, bsp) {
+ const i = abc.memory_array[bsp/4+2];
+
+ /* get arguments from the stack */
+ const s_ptr = abc.memory_array[asp/4];
+ const size = abc.memory_array[s_ptr/4+2];
+ const s = new Uint8Array(abc.memory_array.buffer, s_ptr+16, size);
+
+ this.FS.write(abc.files[i].stream, s, 0, size);
+
+ abc.interpreter.instance.exports.set_asp(asp-8); /* pop the string from the stack */
+ return pc+8; /* return the address of the next instruction */
+}
+```
+
+In reality we need about 10 instructions for file I/O, but the others are
+similar.
+
+# Running the generated bytecode
+
+We can now run a Clean program from our editor through the [compilation
+pipeline][pipeline] and generate a prelinked bytecode file. This is similar to
+a native executable, but for the WebAssembly interpreter. What remains is
+actually running the bytecode.
+
+We do this by simply creating a new instance of the ABC interpreter. The
+`instantiate` function uses `fetch()` to get supporting WebAssembly and
+bytecode files. In this case, our bytecode comes from the local file system, so
+we monkey-patch `fetch()` to supply the bytecode from the Emscripten file
+system (the case that the path is `null`):
+
+```js
+function run (path) {
+ const opts = {
+ fetch: (p) => {
+ return p !== null ? fetch(p) : new Promise(resolve => {
+ const pbc = c_bcprelink.FS.readFile(path);
+ resolve({
+ ok: true,
+ arrayBuffer: () => pbc.buffer
+ });
+ });
+ },
+ };
+
+ return ABCInterpreter.instantiate(opts).then(abc => {
+ abc.interpreter.instance.exports.set_pc(abc.start);
+ var r = 0;
+ try {
+ r = abc.interpreter.instance.exports.interpret();
+ } catch (e) {
+ sandbox.stderr(e+'\n');
+ r = -1;
+ }
+
+ /* flush output buffer */
+ if (sandbox.stdout_buffer.length>0)
+ sandbox.stdout('\n');
+
+ if (r!=0)
+ sandbox.stderr('failed with return code '+r);
+ });
+}
+```
+
+# Wrapping up
+
+That's it! There are many other small tricks here and there, but these four
+posts covered the architecture and most interesting implementation details. You
+can try out the [Clean Sandbox][] to see it live, or check out the [source
+code](https://gitlab.com/camilstaps/clean-sandbox).
+
+[Clean]: http://clean.cs.ru.nl/
+[Clean Sandbox]: https://camilstaps.gitlab.io/clean-sandbox/
+[WebAssembly]: https://webassembly.org/
+
+[introduction]: 2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-1-introduction.html
+[pipeline]: 2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-2-the-pipeline.html
+[integration]: 2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-3-putting-it-all-together.html
diff --git a/resources/md/2021-06-23-running-the-clean-compiler-in-the-browser-with-webassembly.md b/resources/md/2021-06-23-running-the-clean-compiler-in-the-browser-with-webassembly.md
deleted file mode 100644
index 623a080..0000000
--- a/resources/md/2021-06-23-running-the-clean-compiler-in-the-browser-with-webassembly.md
+++ /dev/null
@@ -1,7 +0,0 @@
-Example of an article with markdown.
-
-See https://camilstaps.gitlab.io/clean-sandbox.
-
-```js
-let xyz = 5;
-```
diff --git a/resources/pug/finals/articles/2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-1-introduction.pug b/resources/pug/finals/articles/2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-1-introduction.pug
new file mode 100644
index 0000000..9d5cf91
--- /dev/null
+++ b/resources/pug/finals/articles/2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-1-introduction.pug
@@ -0,0 +1,18 @@
+extends /layout-articles.pug
+
+block prepend title
+ | Compiling Clean in the browser -
+
+block prepend menu
+ - var page = '2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-1-introduction'
+
+block append breadcrumbs
+ +breadcrumb('Compiling Clean in the browser with WebAssembly, part 1: Introduction')
+
+block subtitle
+ | Compiling Clean in the browser with WebAssembly, part 1: Introduction
+block subtitleDate
+ | 23 June 2021
+
+block page
+ include:markdown ../../../md/2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-1-introduction.md
diff --git a/resources/pug/finals/articles/2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-2-the-pipeline.pug b/resources/pug/finals/articles/2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-2-the-pipeline.pug
new file mode 100644
index 0000000..713e97a
--- /dev/null
+++ b/resources/pug/finals/articles/2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-2-the-pipeline.pug
@@ -0,0 +1,18 @@
+extends /layout-articles.pug
+
+block prepend title
+ | Compiling Clean in the browser -
+
+block prepend menu
+ - var page = '2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-2-the-pipeline'
+
+block append breadcrumbs
+ +breadcrumb('Compiling Clean in the browser with WebAssembly, part 2: The pipeline')
+
+block subtitle
+ | Compiling Clean in the browser with WebAssembly, part 2: The pipeline
+block subtitleDate
+ | 23 June 2021
+
+block page
+ include:markdown ../../../md/2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-2-the-pipeline.md
diff --git a/resources/pug/finals/articles/2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-3-putting-it-all-together.pug b/resources/pug/finals/articles/2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-3-putting-it-all-together.pug
new file mode 100644
index 0000000..e684350
--- /dev/null
+++ b/resources/pug/finals/articles/2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-3-putting-it-all-together.pug
@@ -0,0 +1,18 @@
+extends /layout-articles.pug
+
+block prepend title
+ | Compiling Clean in the browser -
+
+block prepend menu
+ - var page = '2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-3-putting-it-all-together'
+
+block append breadcrumbs
+ +breadcrumb('Compiling Clean in the browser with WebAssembly, part 3: Putting it all together')
+
+block subtitle
+ | Compiling Clean in the browser with WebAssembly, part 3: Putting it all together
+block subtitleDate
+ | 23 June 2021
+
+block page
+ include:markdown ../../../md/2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-3-putting-it-all-together.md
diff --git a/resources/pug/finals/articles/2021-06-23-running-the-clean-compiler-in-the-browser-with-webassembly.pug b/resources/pug/finals/articles/2021-06-23-running-the-clean-compiler-in-the-browser-with-webassembly.pug
deleted file mode 100644
index 6506192..0000000
--- a/resources/pug/finals/articles/2021-06-23-running-the-clean-compiler-in-the-browser-with-webassembly.pug
+++ /dev/null
@@ -1,12 +0,0 @@
-extends /layout-articles.pug
-
-block prepend menu
- - var page = '2021-06-23-running-the-clean-compiler-in-the-browser-with-webassembly'
-
-block subtitle
- | Running the Clean compiler in the browser with WebAssembly
-block subtitleDate
- | 23 June 2021
-
-block page
- include:markdown ../../../md/2021-06-23-running-the-clean-compiler-in-the-browser-with-webassembly.md
diff --git a/resources/pug/finals/articles/index.pug b/resources/pug/finals/articles/index.pug
index 3c6f618..b3100dd 100644
--- a/resources/pug/finals/articles/index.pug
+++ b/resources/pug/finals/articles/index.pug
@@ -9,6 +9,16 @@ block subtitle
block subtitleExtra
block page
- p.
- This is a collection of some software-related articles I wrote.
- Perhaps it will be larger than this one day.
+ p
+ | This is a collection of some software-related articles I wrote.
+ | Perhaps it will be larger than this one day.
+ | You can also go back to my #[a(href='/') home page].
+
+ h1
+ a(href='2021-06-23-compiling-clean-in-the-browser-with-webassembly-part-1-introduction.html').
+ 23 June 2021: Compiling Clean in the browser with WebAssembly
+ blockquote.
+ My interpreter for the pure, functional language Clean allows interpretation in both C and WebAssembly.
+ The Clean compiler is written in Clean, with a C backend.
+ In my Clean Sandbox, I compile the C backend with Emscripten to WebAssembly, and interpret the Clean frontend in the WebAssembly interpreter.
+ Combined with a simple make-like tool and an editor, this allows you to compile and run Clean code in the browser.
diff --git a/resources/pug/finals/ham/cw-decoder.pug b/resources/pug/finals/ham/cw-decoder.pug
index b4285eb..ba8d727 100644
--- a/resources/pug/finals/ham/cw-decoder.pug
+++ b/resources/pug/finals/ham/cw-decoder.pug
@@ -4,14 +4,14 @@ block prepend menu
- var page = 'cw-decoder'
- var sections = ['Porting to PIC', 'Finishing', 'Future plans']
+block append breadcrumbs
+ +breadcrumb('CW Decoder')
+
block subtitle
| CW Decoder
block subtitleDate
| January 2017
-block subtitle-right
- +githubIconLink('camilstaps/CWDecoder')
-
block page
h4 Introduction
p.
diff --git a/resources/pug/include/layout-articles.pug b/resources/pug/include/layout-articles.pug
index 2672a36..10feaab 100644
--- a/resources/pug/include/layout-articles.pug
+++ b/resources/pug/include/layout-articles.pug
@@ -1,10 +1,21 @@
extends layout-sidebar.pug
+block append breadcrumbs
+ +breadcrumb('/articles/', 'Articles')
+
block append menu
- var base_url = '/articles/'
+menu(
{name: 'Home', link: ''},
{name: 2021, menu: [
- {name: 'Running the Clean compiler in the browser with WebAssembly', year: 2021, month: 6, day: 23},
+ { name: 'Compiling Clean in the browser with WebAssembly, part 3: Putting it all together',
+ year: 2021, month: 6, day: 23
+ },
+ { name: 'Compiling Clean in the browser with WebAssembly, part 2: The pipeline',
+ year: 2021, month: 6, day: 23
+ },
+ { name: 'Compiling Clean in the browser with WebAssembly, part 1: Introduction',
+ year: 2021, month: 6, day: 23
+ },
]},
)
diff --git a/resources/pug/include/layout-ham.pug b/resources/pug/include/layout-ham.pug
index bb20ef0..9c7ea79 100644
--- a/resources/pug/include/layout-ham.pug
+++ b/resources/pug/include/layout-ham.pug
@@ -3,6 +3,9 @@ extends layout-sidebar.pug
block title
| PA5ET
+block append breadcrumbs
+ +breadcrumb('/ham/', 'Ham Radio')
+
block append menu
- var base_url = '/ham/'
+menu(
diff --git a/resources/pug/include/layout-sidebar.pug b/resources/pug/include/layout-sidebar.pug
index e53476b..9ce6566 100644
--- a/resources/pug/include/layout-sidebar.pug
+++ b/resources/pug/include/layout-sidebar.pug
@@ -18,7 +18,7 @@ block content
function item_link (item) {
if (!('link' in item)) {
- item.link = new String(item.name).toLowerCase().replace(/\W/g, '-');
+ item.link = new String(item.name).toLowerCase().replace(/\W/g, '-').replace(/--+/g, '-');
if ('year' in item && 'month' in item && 'day' in item)
item.link = item.year + '-' + pad_zero(item.month) + '-' + pad_zero(item.day) + '-' + item.link;
@@ -38,7 +38,7 @@ block content
- const id = 'menu-' + item_link(item);
a(href='#' + id, data-toggle='collapse', role='button', aria-expanded=active.toString(), aria-controls=id)= item.name
div.collapse(id=id, class=active ? 'in' : '')
- ul.nav.nav-pills.nav-stacked.col-xs-12.col-md-10.pull-right
+ ul.nav.nav-pills.nav-stacked.col-xs-11.col-md-10.pull-right
+menu(...item.menu)
br(style='clear:both;')
@@ -47,18 +47,30 @@ block content
li(role='presentation', class=item_is_active(item) ? 'active' : '')
a(href=base_url + (link == '' ? '' : link + '.html'))= item.name
- div.col-lg-7.col-md-6
- h3
- block subtitle
- block subtitleExtra
- small
- | by
- block subtitleAuthor
- | Camil Staps
- | ,
- block subtitleDate
- div.col-lg-2.col-md-2.text-right
- block subtitle-right
+ div.col-lg-9.col-md-8
+ ol.breadcrumb.hidden-xs.hidden-sm
+ mixin breadcrumb(link, title)
+ li
+ if typeof title != 'undefined'
+ a(href=link)= title
+ else
+ a(href=base_url + page + '.html')= link
- div.col-lg-9.col-md-8.text-justify.markdown
- block page
+ block breadcrumbs
+ +breadcrumb('/', 'Home')
+
+ div.row.article
+ div.col-lg-12
+ h3
+ strong
+ block subtitle
+ block subtitleExtra
+ small
+ | by
+ block subtitleAuthor
+ | Camil Staps
+ | ,
+ block subtitleDate
+
+ article.col-lg-12.text-justify.markdown
+ block page
diff --git a/resources/pug/include/layout.pug b/resources/pug/include/layout.pug
index afb8c8d..81f133d 100644
--- a/resources/pug/include/layout.pug
+++ b/resources/pug/include/layout.pug
@@ -12,7 +12,6 @@ html(lang="en")
block titleHeading
block title
| Camil Staps
- hr
block content
diff --git a/resources/sass/style.scss b/resources/sass/style.scss
index e401645..1523344 100644
--- a/resources/sass/style.scss
+++ b/resources/sass/style.scss
@@ -96,11 +96,27 @@ span.tt {
* title and subtitle. So we adapt the font size of the headings in the
* markdown content so that they are not larger than the page title. */
.markdown {
+ h1, h2, h3 { font-weight: bold; }
h1 { font-size: 18px; } /* size of h4 */
h2 { font-size: 14px; } /* size of h5 */
h3 { font-size: 12px; } /* size of h5 */
}
+pre code {
+ tab-size: 4;
+ -moz-tab-size: 4;
+ white-space: pre;
+}
+
+pre code.lang-js {
+ tab-size: 2;
+ -moz-tab-size: 2;
+}
+
+blockquote {
+ font-size: unset;
+}
+
footer {
padding: 1em 0;
}
@@ -108,3 +124,7 @@ footer {
.obfuscate > span {
display: none;
}
+
+.breadcrumb {
+ margin-bottom: 0;
+}
--
cgit v1.2.3