Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions LOADING_WASM.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,4 +31,13 @@ import { loadModule } from 'libpg-query';
await loadModule({ wasm: bytes }); // ArrayBuffer, typed array, or a compiled WebAssembly.Module
```

On Cloudflare Workers, import the `.wasm` as a module and pass it in:

```ts
import { loadModule } from 'libpg-query';
import wasm from 'libpg-query/wasm/libpg-query.wasm';

await loadModule({ wasm });
```

Pass only the `libpg-query.wasm` shipped with the package (or bytes you trust as much): whatever you pass runs as the parser.
3 changes: 2 additions & 1 deletion scripts/copy-templates.js
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,8 @@ const TEMPLATE_FILES = [
{ src: 'LICENSE', dest: 'LICENSE', header: false, hasFullVariant: false },
{ src: 'wasm_wrapper.c', dest: 'src/wasm_wrapper.c', header: HEADER, hasFullVariant: true },
{ src: 'libpg-query.d.ts', dest: 'src/libpg-query.d.ts', header: HEADER, hasFullVariant: true },
{ src: 'index.ts', dest: 'src/index.ts', header: HEADER, hasFullVariant: true }
{ src: 'index.ts', dest: 'src/index.ts', header: HEADER, hasFullVariant: true },
{ src: 'pre.js', dest: 'src/pre.js', header: HEADER, hasFullVariant: false }
];

const SLIM_EXPORTED_FUNCTIONS = "['_malloc','_free','_wasm_parse_query_raw','_wasm_free_parse_result']";
Expand Down
4 changes: 3 additions & 1 deletion templates/Makefile.template
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ endif

PLATFORM_ARCH := $(PLATFORM)-$(ARCH)
SRC_FILES := src/wasm_wrapper.c
PRE_JS := src/pre.js
LIBPG_QUERY_DIR := $(CACHE_DIR)/$(PLATFORM_ARCH)/libpg_query/$(LIBPG_QUERY_COMMIT)
LIBPG_QUERY_ARCHIVE := $(LIBPG_QUERY_DIR)/libpg_query.a
LIBPG_QUERY_HEADER := $(LIBPG_QUERY_DIR)/pg_query.h
Expand Down Expand Up @@ -68,7 +69,7 @@ $(LIBPG_QUERY_ARCHIVE): $(LIBPG_QUERY_DIR)
cd $(LIBPG_QUERY_DIR); $(MAKE) build

# Build libpg-query-node WASM module
$(OUT_FILES): $(LIBPG_QUERY_ARCHIVE) $(LIBPG_QUERY_HEADER) $(SRC_FILES)
$(OUT_FILES): $(LIBPG_QUERY_ARCHIVE) $(LIBPG_QUERY_HEADER) $(SRC_FILES) $(PRE_JS)
ifdef EMSCRIPTEN
mkdir -p $(WASM_OUT_DIR)
$(CC) \
Expand All @@ -81,6 +82,7 @@ ifdef EMSCRIPTEN
-sEXPORTED_RUNTIME_METHODS="['lengthBytesUTF8','stringToUTF8','getValue','UTF8ToString','HEAPU8','HEAPU32']" \
-sEXPORT_NAME="$(WASM_MODULE_NAME)" \
-sENVIRONMENT="web,node,worker" \
--pre-js $(PRE_JS) \
-sASSERTIONS=0 \
-sSINGLE_FILE=0 \
-sMODULARIZE=1 \
Expand Down
7 changes: 7 additions & 0 deletions templates/pre.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
// The glue picks Node from `process.versions.node` and a worker from
// `WorkerGlobalScope`, then reads `__dirname` or `self.location` for them.
// Hosts that fake those markers without the rest (Cloudflare Workers with
// `nodejs_compat`, CommonJS bundled into ESM) would throw on the read, before
// `loadModule({ wasm })` can use the bytes it was given.
if (ENVIRONMENT_IS_NODE && typeof __dirname == 'undefined') ENVIRONMENT_IS_NODE = false;
if (ENVIRONMENT_IS_WORKER && !globalThis.self?.location) ENVIRONMENT_IS_WORKER = false;
4 changes: 3 additions & 1 deletion versions/13/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ endif

PLATFORM_ARCH := $(PLATFORM)-$(ARCH)
SRC_FILES := src/wasm_wrapper.c
PRE_JS := src/pre.js
LIBPG_QUERY_DIR := $(CACHE_DIR)/$(PLATFORM_ARCH)/libpg_query/$(LIBPG_QUERY_COMMIT)
LIBPG_QUERY_ARCHIVE := $(LIBPG_QUERY_DIR)/libpg_query.a
LIBPG_QUERY_HEADER := $(LIBPG_QUERY_DIR)/pg_query.h
Expand Down Expand Up @@ -72,7 +73,7 @@ $(LIBPG_QUERY_ARCHIVE): $(LIBPG_QUERY_DIR)
cd $(LIBPG_QUERY_DIR); $(MAKE) build

# Build libpg-query-node WASM module
$(OUT_FILES): $(LIBPG_QUERY_ARCHIVE) $(LIBPG_QUERY_HEADER) $(SRC_FILES)
$(OUT_FILES): $(LIBPG_QUERY_ARCHIVE) $(LIBPG_QUERY_HEADER) $(SRC_FILES) $(PRE_JS)
ifdef EMSCRIPTEN
mkdir -p $(WASM_OUT_DIR)
$(CC) \
Expand All @@ -85,6 +86,7 @@ ifdef EMSCRIPTEN
-sEXPORTED_RUNTIME_METHODS="['lengthBytesUTF8','stringToUTF8','getValue','UTF8ToString','HEAPU8','HEAPU32']" \
-sEXPORT_NAME="$(WASM_MODULE_NAME)" \
-sENVIRONMENT="web,node,worker" \
--pre-js $(PRE_JS) \
-sASSERTIONS=0 \
-sSINGLE_FILE=0 \
-sMODULARIZE=1 \
Expand Down
14 changes: 14 additions & 0 deletions versions/13/src/pre.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
/**
* DO NOT MODIFY MANUALLY — this is generated from the templates dir
*
* To make changes, edit the files in the templates/ directory and run:
* npm run copy:templates
*/

// The glue picks Node from `process.versions.node` and a worker from
// `WorkerGlobalScope`, then reads `__dirname` or `self.location` for them.
// Hosts that fake those markers without the rest (Cloudflare Workers with
// `nodejs_compat`, CommonJS bundled into ESM) would throw on the read, before
// `loadModule({ wasm })` can use the bytes it was given.
if (ENVIRONMENT_IS_NODE && typeof __dirname == 'undefined') ENVIRONMENT_IS_NODE = false;
if (ENVIRONMENT_IS_WORKER && !globalThis.self?.location) ENVIRONMENT_IS_WORKER = false;
54 changes: 54 additions & 0 deletions versions/13/test/load-module.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ const assert = require('node:assert/strict');
const { spawnSync } = require('node:child_process');
const fs = require('node:fs');
const path = require('node:path');
const os = require('node:os');

const WASM = path.join(__dirname, '../wasm/libpg-query.wasm');
const RESULT = 'RESULT ';
Expand Down Expand Up @@ -92,6 +93,59 @@ describe('loadModule({ wasm }) when the wrapper cannot read libpg-query.wasm', {
});
});

// Cloudflare Workers with `nodejs_compat` define `process.versions.node` and
// `WorkerGlobalScope`, but no `__dirname`, `__filename` or `self.location`.
// Loads the wrapper and the glue as plain functions, so neither sees a module
// scope, in a process that also has those two markers.
function inWorkerLikeHost(body) {
const dir = path.join(__dirname, '../wasm');
const script = `
globalThis.WorkerGlobalScope = class WorkerGlobalScope {};
globalThis.self = globalThis;
const fs = require('node:fs');
const path = require('node:path');
const dir = ${JSON.stringify(dir)};
const load = (file, req) => {
const module = { exports: {} };
new Function('module', 'exports', 'require', fs.readFileSync(path.join(dir, file), 'utf8'))(module, module.exports, req);
return module.exports;
};
const glue = load('libpg-query.js', (id) => { throw new Error('the glue required ' + id); });
const lib = load('index.cjs', (id) => id === './libpg-query.js' ? glue : require(require.resolve(id, { paths: [dir] })));
const bytes = fs.readFileSync(path.join(dir, 'libpg-query.wasm'));
(async () => { ${body} })().then(
(result) => console.log(${JSON.stringify(RESULT)} + JSON.stringify(result)),
(error) => { console.log(${JSON.stringify(RESULT)} + JSON.stringify({ error: String(error && error.message || error) })); }
);
`;
// A file, not `-e`: eval code gets `__dirname` and `__filename` as globals.
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'worker-like-'));
const file = path.join(tmp, 'host.js');
fs.writeFileSync(file, script);
let run;
try {
run = spawnSync(process.execPath, [file], { encoding: 'utf8', timeout: 120000 });
} finally {
fs.rmSync(tmp, { recursive: true, force: true });
}
const line = (run.stdout || '').split('\n').reverse().find((l) => l.startsWith(RESULT));
if (!line) {
const how = run.error ? run.error.message : run.signal ? `signal ${run.signal}` : `exit ${run.status}`;
throw new Error(`child process gave no result (${how}):\n${(run.stderr || '').slice(-4000)}`);
}
return JSON.parse(line.slice(RESULT.length));
}

describe('loadModule({ wasm }) in a host without __dirname or self.location', { timeout: 300000 }, () => {
it('loads from the bytes and parses', () => {
const result = inWorkerLikeHost(`
await lib.loadModule({ wasm: bytes });
return { stmts: (await lib.parse('SELECT 1')).stmts.length };
`);
assert.deepEqual(result, { stmts: 1 });
});
});

describe('loadModule() arity', () => {
it('stays 0 so test runners can pass it directly as a hook', () => {
assert.equal(require('../wasm/index.cjs').loadModule.length, 0);
Expand Down
4 changes: 3 additions & 1 deletion versions/14/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ endif

PLATFORM_ARCH := $(PLATFORM)-$(ARCH)
SRC_FILES := src/wasm_wrapper.c
PRE_JS := src/pre.js
LIBPG_QUERY_DIR := $(CACHE_DIR)/$(PLATFORM_ARCH)/libpg_query/$(LIBPG_QUERY_COMMIT)
LIBPG_QUERY_ARCHIVE := $(LIBPG_QUERY_DIR)/libpg_query.a
LIBPG_QUERY_HEADER := $(LIBPG_QUERY_DIR)/pg_query.h
Expand All @@ -67,7 +68,7 @@ $(LIBPG_QUERY_ARCHIVE): $(LIBPG_QUERY_DIR)
cd $(LIBPG_QUERY_DIR); $(MAKE) build

# Build libpg-query-node WASM module
$(OUT_FILES): $(LIBPG_QUERY_ARCHIVE) $(LIBPG_QUERY_HEADER) $(SRC_FILES)
$(OUT_FILES): $(LIBPG_QUERY_ARCHIVE) $(LIBPG_QUERY_HEADER) $(SRC_FILES) $(PRE_JS)
ifdef EMSCRIPTEN
mkdir -p $(WASM_OUT_DIR)
$(CC) \
Expand All @@ -80,6 +81,7 @@ ifdef EMSCRIPTEN
-sEXPORTED_RUNTIME_METHODS="['lengthBytesUTF8','stringToUTF8','getValue','UTF8ToString','HEAPU8','HEAPU32']" \
-sEXPORT_NAME="$(WASM_MODULE_NAME)" \
-sENVIRONMENT="web,node,worker" \
--pre-js $(PRE_JS) \
-sASSERTIONS=0 \
-sSINGLE_FILE=0 \
-sMODULARIZE=1 \
Expand Down
14 changes: 14 additions & 0 deletions versions/14/src/pre.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
/**
* DO NOT MODIFY MANUALLY — this is generated from the templates dir
*
* To make changes, edit the files in the templates/ directory and run:
* npm run copy:templates
*/

// The glue picks Node from `process.versions.node` and a worker from
// `WorkerGlobalScope`, then reads `__dirname` or `self.location` for them.
// Hosts that fake those markers without the rest (Cloudflare Workers with
// `nodejs_compat`, CommonJS bundled into ESM) would throw on the read, before
// `loadModule({ wasm })` can use the bytes it was given.
if (ENVIRONMENT_IS_NODE && typeof __dirname == 'undefined') ENVIRONMENT_IS_NODE = false;
if (ENVIRONMENT_IS_WORKER && !globalThis.self?.location) ENVIRONMENT_IS_WORKER = false;
54 changes: 54 additions & 0 deletions versions/14/test/load-module.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ const assert = require('node:assert/strict');
const { spawnSync } = require('node:child_process');
const fs = require('node:fs');
const path = require('node:path');
const os = require('node:os');

const WASM = path.join(__dirname, '../wasm/libpg-query.wasm');
const RESULT = 'RESULT ';
Expand Down Expand Up @@ -92,6 +93,59 @@ describe('loadModule({ wasm }) when the wrapper cannot read libpg-query.wasm', {
});
});

// Cloudflare Workers with `nodejs_compat` define `process.versions.node` and
// `WorkerGlobalScope`, but no `__dirname`, `__filename` or `self.location`.
// Loads the wrapper and the glue as plain functions, so neither sees a module
// scope, in a process that also has those two markers.
function inWorkerLikeHost(body) {
const dir = path.join(__dirname, '../wasm');
const script = `
globalThis.WorkerGlobalScope = class WorkerGlobalScope {};
globalThis.self = globalThis;
const fs = require('node:fs');
const path = require('node:path');
const dir = ${JSON.stringify(dir)};
const load = (file, req) => {
const module = { exports: {} };
new Function('module', 'exports', 'require', fs.readFileSync(path.join(dir, file), 'utf8'))(module, module.exports, req);
return module.exports;
};
const glue = load('libpg-query.js', (id) => { throw new Error('the glue required ' + id); });
const lib = load('index.cjs', (id) => id === './libpg-query.js' ? glue : require(require.resolve(id, { paths: [dir] })));
const bytes = fs.readFileSync(path.join(dir, 'libpg-query.wasm'));
(async () => { ${body} })().then(
(result) => console.log(${JSON.stringify(RESULT)} + JSON.stringify(result)),
(error) => { console.log(${JSON.stringify(RESULT)} + JSON.stringify({ error: String(error && error.message || error) })); }
);
`;
// A file, not `-e`: eval code gets `__dirname` and `__filename` as globals.
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'worker-like-'));
const file = path.join(tmp, 'host.js');
fs.writeFileSync(file, script);
let run;
try {
run = spawnSync(process.execPath, [file], { encoding: 'utf8', timeout: 120000 });
} finally {
fs.rmSync(tmp, { recursive: true, force: true });
}
const line = (run.stdout || '').split('\n').reverse().find((l) => l.startsWith(RESULT));
if (!line) {
const how = run.error ? run.error.message : run.signal ? `signal ${run.signal}` : `exit ${run.status}`;
throw new Error(`child process gave no result (${how}):\n${(run.stderr || '').slice(-4000)}`);
}
return JSON.parse(line.slice(RESULT.length));
}

describe('loadModule({ wasm }) in a host without __dirname or self.location', { timeout: 300000 }, () => {
it('loads from the bytes and parses', () => {
const result = inWorkerLikeHost(`
await lib.loadModule({ wasm: bytes });
return { stmts: (await lib.parse('SELECT 1')).stmts.length };
`);
assert.deepEqual(result, { stmts: 1 });
});
});

describe('loadModule() arity', () => {
it('stays 0 so test runners can pass it directly as a hook', () => {
assert.equal(require('../wasm/index.cjs').loadModule.length, 0);
Expand Down
4 changes: 3 additions & 1 deletion versions/15/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ endif

PLATFORM_ARCH := $(PLATFORM)-$(ARCH)
SRC_FILES := src/wasm_wrapper.c
PRE_JS := src/pre.js
LIBPG_QUERY_DIR := $(CACHE_DIR)/$(PLATFORM_ARCH)/libpg_query/$(LIBPG_QUERY_COMMIT)
LIBPG_QUERY_ARCHIVE := $(LIBPG_QUERY_DIR)/libpg_query.a
LIBPG_QUERY_HEADER := $(LIBPG_QUERY_DIR)/pg_query.h
Expand All @@ -67,7 +68,7 @@ $(LIBPG_QUERY_ARCHIVE): $(LIBPG_QUERY_DIR)
cd $(LIBPG_QUERY_DIR); $(MAKE) build

# Build libpg-query-node WASM module
$(OUT_FILES): $(LIBPG_QUERY_ARCHIVE) $(LIBPG_QUERY_HEADER) $(SRC_FILES)
$(OUT_FILES): $(LIBPG_QUERY_ARCHIVE) $(LIBPG_QUERY_HEADER) $(SRC_FILES) $(PRE_JS)
ifdef EMSCRIPTEN
mkdir -p $(WASM_OUT_DIR)
$(CC) \
Expand All @@ -80,6 +81,7 @@ ifdef EMSCRIPTEN
-sEXPORTED_RUNTIME_METHODS="['lengthBytesUTF8','stringToUTF8','getValue','UTF8ToString','HEAPU8','HEAPU32']" \
-sEXPORT_NAME="$(WASM_MODULE_NAME)" \
-sENVIRONMENT="web,node,worker" \
--pre-js $(PRE_JS) \
-sASSERTIONS=0 \
-sSINGLE_FILE=0 \
-sMODULARIZE=1 \
Expand Down
14 changes: 14 additions & 0 deletions versions/15/src/pre.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
/**
* DO NOT MODIFY MANUALLY — this is generated from the templates dir
*
* To make changes, edit the files in the templates/ directory and run:
* npm run copy:templates
*/

// The glue picks Node from `process.versions.node` and a worker from
// `WorkerGlobalScope`, then reads `__dirname` or `self.location` for them.
// Hosts that fake those markers without the rest (Cloudflare Workers with
// `nodejs_compat`, CommonJS bundled into ESM) would throw on the read, before
// `loadModule({ wasm })` can use the bytes it was given.
if (ENVIRONMENT_IS_NODE && typeof __dirname == 'undefined') ENVIRONMENT_IS_NODE = false;
if (ENVIRONMENT_IS_WORKER && !globalThis.self?.location) ENVIRONMENT_IS_WORKER = false;
54 changes: 54 additions & 0 deletions versions/15/test/load-module.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ const assert = require('node:assert/strict');
const { spawnSync } = require('node:child_process');
const fs = require('node:fs');
const path = require('node:path');
const os = require('node:os');

const WASM = path.join(__dirname, '../wasm/libpg-query.wasm');
const RESULT = 'RESULT ';
Expand Down Expand Up @@ -92,6 +93,59 @@ describe('loadModule({ wasm }) when the wrapper cannot read libpg-query.wasm', {
});
});

// Cloudflare Workers with `nodejs_compat` define `process.versions.node` and
// `WorkerGlobalScope`, but no `__dirname`, `__filename` or `self.location`.
// Loads the wrapper and the glue as plain functions, so neither sees a module
// scope, in a process that also has those two markers.
function inWorkerLikeHost(body) {
const dir = path.join(__dirname, '../wasm');
const script = `
globalThis.WorkerGlobalScope = class WorkerGlobalScope {};
globalThis.self = globalThis;
const fs = require('node:fs');
const path = require('node:path');
const dir = ${JSON.stringify(dir)};
const load = (file, req) => {
const module = { exports: {} };
new Function('module', 'exports', 'require', fs.readFileSync(path.join(dir, file), 'utf8'))(module, module.exports, req);
return module.exports;
};
const glue = load('libpg-query.js', (id) => { throw new Error('the glue required ' + id); });
const lib = load('index.cjs', (id) => id === './libpg-query.js' ? glue : require(require.resolve(id, { paths: [dir] })));
const bytes = fs.readFileSync(path.join(dir, 'libpg-query.wasm'));
(async () => { ${body} })().then(
(result) => console.log(${JSON.stringify(RESULT)} + JSON.stringify(result)),
(error) => { console.log(${JSON.stringify(RESULT)} + JSON.stringify({ error: String(error && error.message || error) })); }
);
`;
// A file, not `-e`: eval code gets `__dirname` and `__filename` as globals.
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'worker-like-'));
const file = path.join(tmp, 'host.js');
fs.writeFileSync(file, script);
let run;
try {
run = spawnSync(process.execPath, [file], { encoding: 'utf8', timeout: 120000 });
} finally {
fs.rmSync(tmp, { recursive: true, force: true });
}
const line = (run.stdout || '').split('\n').reverse().find((l) => l.startsWith(RESULT));
if (!line) {
const how = run.error ? run.error.message : run.signal ? `signal ${run.signal}` : `exit ${run.status}`;
throw new Error(`child process gave no result (${how}):\n${(run.stderr || '').slice(-4000)}`);
}
return JSON.parse(line.slice(RESULT.length));
}

describe('loadModule({ wasm }) in a host without __dirname or self.location', { timeout: 300000 }, () => {
it('loads from the bytes and parses', () => {
const result = inWorkerLikeHost(`
await lib.loadModule({ wasm: bytes });
return { stmts: (await lib.parse('SELECT 1')).stmts.length };
`);
assert.deepEqual(result, { stmts: 1 });
});
});

describe('loadModule() arity', () => {
it('stays 0 so test runners can pass it directly as a hook', () => {
assert.equal(require('../wasm/index.cjs').loadModule.length, 0);
Expand Down
Loading
Loading