KurlesHS/rclone
1
0
Fork 0
forked from TrueCloudLab/rclone
Code Pull requests Activity

js: add experimental interface for integrating rclone into browsers

Browse source 
This works by compiling rclone to wasm and exporting the RC api to
javascript.
This commit is contained in:
Nick Craig-Wood 2020-08-01 15:36:20 +01:00
parent 3a14b1d5a9
commit aab9aa8a2e
8 changed files with 830 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
fs/rc/js/.gitignore vendored Normal file
View file

@ -0,0 +1 @@
*.wasm

5
fs/rc/js/Makefile Normal file
View file

@ -0,0 +1,5 @@
build:
GOARCH=wasm GOOS=js go build -o rclone.wasm
serve: build
go run serve.go

25
fs/rc/js/README.md Normal file
View file

@ -0,0 +1,25 @@
# Rclone as WASM
This directory contains files to use the rclone rc as a library in the browser.
This works by compiling rclone to WASM and loading that in via javascript.
This contains the following files
- `index.html` - test web page to load the module
- `loader.js` - java script to load the module - see here for usage
- `main.go` - main go code exporting the rclone rc
- `Makefile` - test makefile
- `README.md` - this readme
- `serve.go` - test program to serve the web page
- `wasm_exec.js` - interface code from the go source - don't edit
## Compiling
This can be compiled by using `make` or alternatively `GOARCH=wasm GOOS=js go build -o rclone.wasm`
## Running
Run the test server with `make serve` and examine the page at
http://localhost:3000/ - look at the javascript console and look at
the end of `loader.js` for how that works.

11
fs/rc/js/index.html Normal file
View file

@ -0,0 +1,11 @@
<!doctype html>
<html>
<head>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Rclone</title>
</head>
<body>
<script src="loader.js"></script>
<p>Welcome to rclone - check the console</p>
</body>
</html>

36
fs/rc/js/loader.js Normal file
View file

@ -0,0 +1,36 @@
var rc;
var rcValidResolve
var rcValid = new Promise(resolve => {
rcValidResolve = resolve
});
var script = document.createElement('script');
script.src = "wasm_exec.js";
script.onload = function () {
if (!WebAssembly.instantiateStreaming) { // polyfill
WebAssembly.instantiateStreaming = async (resp, importObject) => {
const source = await (await resp).arrayBuffer();
return await WebAssembly.instantiate(source, importObject);
};
}
const go = new Go();
WebAssembly.instantiateStreaming(fetch("rclone.wasm"), go.importObject).then((result) => {
go.run(result.instance);
});
};
document.head.appendChild(script);
rcValid.then(() => {
// Some examples of using the rc call
//
// The rc call takes two parameters, method and input object and
// returns an output object.
//
// If the output object has an "error" and a "status" then it is an
// error (it would be nice to signal this out of band).
console.log("core/version", rc("core/version", null))
console.log("core/version", rc("rc/noop", {"string":"one",number:2}))
console.log("core/version", rc("operations/mkdir", {"fs":":memory:","remote":"bucket"}))
console.log("core/version", rc("operations/list", {"fs":":memory:","remote":"bucket"}))
})

145
fs/rc/js/main.go Normal file
View file

@ -0,0 +1,145 @@
// Rclone as a wasm library
//
// This library exports the core rc functionality
// +build js
package main
import (
"context"
"encoding/json"
"log"
"net/http"
"runtime"
"syscall/js"
"github.com/pkg/errors"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/rc"
// Core functionality we need
_ "github.com/rclone/rclone/fs/operations"
_ "github.com/rclone/rclone/fs/sync"
// _ "github.com/rclone/rclone/backend/all" // import all backends
// Backends
_ "github.com/rclone/rclone/backend/memory"
)
var (
document js.Value
jsJSON js.Value
)
func getElementById(name string) js.Value {
node := document.Call("getElementById", name)
if node.IsUndefined() {
log.Fatalf("Couldn't find element %q", name)
}
return node
}
func time() int {
return js.Global().Get("Date").New().Call("getTime").Int()
}
func paramToValue(in rc.Params) (out js.Value) {
return js.Value{}
}
// errorValue turns an error into a js.Value
func errorValue(method string, in js.Value, err error) js.Value {
fs.Errorf(nil, "rc: %q: error: %v", method, err)
// Adjust the error return for some well known errors
errOrig := errors.Cause(err)
status := http.StatusInternalServerError
switch {
case errOrig == fs.ErrorDirNotFound || errOrig == fs.ErrorObjectNotFound:
status = http.StatusNotFound
case rc.IsErrParamInvalid(err) || rc.IsErrParamNotFound(err):
status = http.StatusBadRequest
}
return js.ValueOf(map[string]interface{}{
"status": status,
"error": err.Error(),
"input": in,
"path": method,
})
}
// rcCallback is a callback for javascript to access the api
//
// FIXME should this should return a promise so we can return errors properly?
func rcCallback(this js.Value, args []js.Value) interface{} {
ctx := context.Background() // FIXME
log.Printf("rcCallback: this=%v args=%v", this, args)
if len(args) != 2 {
return errorValue("", js.Undefined(), errors.New("need two parameters to rc call"))
}
method := args[0].String()
inRaw := args[1]
var in = rc.Params{}
switch inRaw.Type() {
case js.TypeNull:
case js.TypeObject:
inJSON := jsJSON.Call("stringify", inRaw).String()
err := json.Unmarshal([]byte(inJSON), &in)
if err != nil {
return errorValue(method, inRaw, errors.Wrap(err, "couldn't unmarshal input"))
}
default:
return errorValue(method, inRaw, errors.New("in parameter must be null or object"))
}
call := rc.Calls.Get(method)
if call == nil {
return errorValue(method, inRaw, errors.Errorf("method %q not found", method))
}
out, err := call.Fn(ctx, in)
if err != nil {
return errorValue(method, inRaw, errors.Wrap(err, "method call failed"))
}
if out == nil {
return nil
}
var out2 map[string]interface{}
err = rc.Reshape(&out2, out)
if err != nil {
return errorValue(method, inRaw, errors.Wrap(err, "result reshape failed"))
}
return js.ValueOf(out2)
}
func main() {
log.Printf("Running on goos/goarch = %s/%s", runtime.GOOS, runtime.GOARCH)
if js.Global().IsUndefined() {
log.Fatalf("Didn't find Global - not running in browser")
}
document = js.Global().Get("document")
if document.IsUndefined() {
log.Fatalf("Didn't find document - not running in browser")
}
jsJSON = js.Global().Get("JSON")
if jsJSON.IsUndefined() {
log.Fatalf("can't find JSON")
}
// Set rc
js.Global().Set("rc", js.FuncOf(rcCallback))
// Signal that it is valid
rcValidResolve := js.Global().Get("rcValidResolve")
if rcValidResolve.IsUndefined() {
log.Fatalf("Didn't find rcValidResolve")
}
rcValidResolve.Invoke()
// Wait forever
select {}
}

19
fs/rc/js/serve.go Normal file
View file

@ -0,0 +1,19 @@
//+build none
package main
import (
"fmt"
"log"
"mime"
"net/http"
)
func main() {
mime.AddExtensionType(".wasm", "application/wasm")
mime.AddExtensionType(".js", "application/javascript")
mux := http.NewServeMux()
mux.Handle("/", http.FileServer(http.Dir(".")))
fmt.Printf("Serving on http://localhost:3000/\n")
log.Fatal(http.ListenAndServe(":3000", mux))
}

588
fs/rc/js/wasm_exec.js Normal file
View file

@ -0,0 +1,588 @@
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
(() => {
// Map multiple JavaScript environments to a single common API,
// preferring web standards over Node.js API.
//
// Environments considered:
// - Browsers
// - Node.js
// - Electron
// - Parcel
if (typeof global !== "undefined") {
// global already exists
} else if (typeof window !== "undefined") {
window.global = window;
} else if (typeof self !== "undefined") {
self.global = self;
} else {
throw new Error("cannot export Go (neither global, window nor self is defined)");
}
if (!global.require && typeof require !== "undefined") {
global.require = require;
}
if (!global.fs && global.require) {
const fs = require("fs");
if (Object.keys(fs) !== 0) {
global.fs = fs;
}
}
const enosys = () => {
const err = new Error("not implemented");
err.code = "ENOSYS";
return err;
};
if (!global.fs) {
let outputBuf = "";
global.fs = {
constants: { O_WRONLY: -1, O_RDWR: -1, O_CREAT: -1, O_TRUNC: -1, O_APPEND: -1, O_EXCL: -1 }, // unused
writeSync(fd, buf) {
outputBuf += decoder.decode(buf);
const nl = outputBuf.lastIndexOf("\n");
if (nl != -1) {
console.log(outputBuf.substr(0, nl));
outputBuf = outputBuf.substr(nl + 1);
}
return buf.length;
},
write(fd, buf, offset, length, position, callback) {
if (offset !== 0 || length !== buf.length || position !== null) {
callback(enosys());
return;
}
const n = this.writeSync(fd, buf);
callback(null, n);
},
chmod(path, mode, callback) { callback(enosys()); },
chown(path, uid, gid, callback) { callback(enosys()); },
close(fd, callback) { callback(enosys()); },
fchmod(fd, mode, callback) { callback(enosys()); },
fchown(fd, uid, gid, callback) { callback(enosys()); },
fstat(fd, callback) { callback(enosys()); },
fsync(fd, callback) { callback(null); },
ftruncate(fd, length, callback) { callback(enosys()); },
lchown(path, uid, gid, callback) { callback(enosys()); },
link(path, link, callback) { callback(enosys()); },
lstat(path, callback) { callback(enosys()); },
mkdir(path, perm, callback) { callback(enosys()); },
open(path, flags, mode, callback) { callback(enosys()); },
read(fd, buffer, offset, length, position, callback) { callback(enosys()); },
readdir(path, callback) { callback(enosys()); },
readlink(path, callback) { callback(enosys()); },
rename(from, to, callback) { callback(enosys()); },
rmdir(path, callback) { callback(enosys()); },
stat(path, callback) { callback(enosys()); },
symlink(path, link, callback) { callback(enosys()); },
truncate(path, length, callback) { callback(enosys()); },
unlink(path, callback) { callback(enosys()); },
utimes(path, atime, mtime, callback) { callback(enosys()); },
};
}
if (!global.process) {
global.process = {
getuid() { return -1; },
getgid() { return -1; },
geteuid() { return -1; },
getegid() { return -1; },
getgroups() { throw enosys(); },
pid: -1,
ppid: -1,
umask() { throw enosys(); },
cwd() { throw enosys(); },
chdir() { throw enosys(); },
}
}
if (!global.crypto) {
const nodeCrypto = require("crypto");
global.crypto = {
getRandomValues(b) {
nodeCrypto.randomFillSync(b);
},
};
}
if (!global.performance) {
global.performance = {
now() {
const [sec, nsec] = process.hrtime();
return sec * 1000 + nsec / 1000000;
},
};
}
if (!global.TextEncoder) {
global.TextEncoder = require("util").TextEncoder;
}
if (!global.TextDecoder) {
global.TextDecoder = require("util").TextDecoder;
}
// End of polyfills for common API.
const encoder = new TextEncoder("utf-8");
const decoder = new TextDecoder("utf-8");
global.Go = class {
constructor() {
this.argv = ["js"];
this.env = {};
this.exit = (code) => {
if (code !== 0) {
console.warn("exit code:", code);
}
};
this._exitPromise = new Promise((resolve) => {
this._resolveExitPromise = resolve;
});
this._pendingEvent = null;
this._scheduledTimeouts = new Map();
this._nextCallbackTimeoutID = 1;
const setInt64 = (addr, v) => {
this.mem.setUint32(addr + 0, v, true);
this.mem.setUint32(addr + 4, Math.floor(v / 4294967296), true);
}
const getInt64 = (addr) => {
const low = this.mem.getUint32(addr + 0, true);
const high = this.mem.getInt32(addr + 4, true);
return low + high * 4294967296;
}
const loadValue = (addr) => {
const f = this.mem.getFloat64(addr, true);
if (f === 0) {
return undefined;
}
if (!isNaN(f)) {
return f;
}
const id = this.mem.getUint32(addr, true);
return this._values[id];
}
const storeValue = (addr, v) => {
const nanHead = 0x7FF80000;
if (typeof v === "number" && v !== 0) {
if (isNaN(v)) {
this.mem.setUint32(addr + 4, nanHead, true);
this.mem.setUint32(addr, 0, true);
return;
}
this.mem.setFloat64(addr, v, true);
return;
}
if (v === undefined) {
this.mem.setFloat64(addr, 0, true);
return;
}
let id = this._ids.get(v);
if (id === undefined) {
id = this._idPool.pop();
if (id === undefined) {
id = this._values.length;
}
this._values[id] = v;
this._goRefCounts[id] = 0;
this._ids.set(v, id);
}
this._goRefCounts[id]++;
let typeFlag = 0;
switch (typeof v) {
case "object":
if (v !== null) {
typeFlag = 1;
}
break;
case "string":
typeFlag = 2;
break;
case "symbol":
typeFlag = 3;
break;
case "function":
typeFlag = 4;
break;
}
this.mem.setUint32(addr + 4, nanHead | typeFlag, true);
this.mem.setUint32(addr, id, true);
}
const loadSlice = (addr) => {
const array = getInt64(addr + 0);
const len = getInt64(addr + 8);
return new Uint8Array(this._inst.exports.mem.buffer, array, len);
}
const loadSliceOfValues = (addr) => {
const array = getInt64(addr + 0);
const len = getInt64(addr + 8);
const a = new Array(len);
for (let i = 0; i < len; i++) {
a[i] = loadValue(array + i * 8);
}
return a;
}
const loadString = (addr) => {
const saddr = getInt64(addr + 0);
const len = getInt64(addr + 8);
return decoder.decode(new DataView(this._inst.exports.mem.buffer, saddr, len));
}
const timeOrigin = Date.now() - performance.now();
this.importObject = {
go: {
// Go's SP does not change as long as no Go code is running. Some operations (e.g. calls, getters and setters)
// may synchronously trigger a Go event handler. This makes Go code get executed in the middle of the imported
// function. A goroutine can switch to a new stack if the current stack is too small (see morestack function).
// This changes the SP, thus we have to update the SP used by the imported function.
// func wasmExit(code int32)
"runtime.wasmExit": (sp) => {
const code = this.mem.getInt32(sp + 8, true);
this.exited = true;
delete this._inst;
delete this._values;
delete this._goRefCounts;
delete this._ids;
delete this._idPool;
this.exit(code);
},
// func wasmWrite(fd uintptr, p unsafe.Pointer, n int32)
"runtime.wasmWrite": (sp) => {
const fd = getInt64(sp + 8);
const p = getInt64(sp + 16);
const n = this.mem.getInt32(sp + 24, true);
fs.writeSync(fd, new Uint8Array(this._inst.exports.mem.buffer, p, n));
},
// func resetMemoryDataView()
"runtime.resetMemoryDataView": (sp) => {
this.mem = new DataView(this._inst.exports.mem.buffer);
},
// func nanotime1() int64
"runtime.nanotime1": (sp) => {
setInt64(sp + 8, (timeOrigin + performance.now()) * 1000000);
},
// func walltime1() (sec int64, nsec int32)
"runtime.walltime1": (sp) => {
const msec = (new Date).getTime();
setInt64(sp + 8, msec / 1000);
this.mem.setInt32(sp + 16, (msec % 1000) * 1000000, true);
},
// func scheduleTimeoutEvent(delay int64) int32
"runtime.scheduleTimeoutEvent": (sp) => {
const id = this._nextCallbackTimeoutID;
this._nextCallbackTimeoutID++;
this._scheduledTimeouts.set(id, setTimeout(
() => {
this._resume();
while (this._scheduledTimeouts.has(id)) {
// for some reason Go failed to register the timeout event, log and try again
// (temporary workaround for https://github.com/golang/go/issues/28975)
console.warn("scheduleTimeoutEvent: missed timeout event");
this._resume();
}
},
getInt64(sp + 8) + 1, // setTimeout has been seen to fire up to 1 millisecond early
));
this.mem.setInt32(sp + 16, id, true);
},
// func clearTimeoutEvent(id int32)
"runtime.clearTimeoutEvent": (sp) => {
const id = this.mem.getInt32(sp + 8, true);
clearTimeout(this._scheduledTimeouts.get(id));
this._scheduledTimeouts.delete(id);
},
// func getRandomData(r []byte)
"runtime.getRandomData": (sp) => {
crypto.getRandomValues(loadSlice(sp + 8));
},
// func finalizeRef(v ref)
"syscall/js.finalizeRef": (sp) => {
const id = this.mem.getUint32(sp + 8, true);
this._goRefCounts[id]--;
if (this._goRefCounts[id] === 0) {
const v = this._values[id];
this._values[id] = null;
this._ids.delete(v);
this._idPool.push(id);
}
},
// func stringVal(value string) ref
"syscall/js.stringVal": (sp) => {
storeValue(sp + 24, loadString(sp + 8));
},
// func valueGet(v ref, p string) ref
"syscall/js.valueGet": (sp) => {
const result = Reflect.get(loadValue(sp + 8), loadString(sp + 16));
sp = this._inst.exports.getsp(); // see comment above
storeValue(sp + 32, result);
},
// func valueSet(v ref, p string, x ref)
"syscall/js.valueSet": (sp) => {
Reflect.set(loadValue(sp + 8), loadString(sp + 16), loadValue(sp + 32));
},
// func valueDelete(v ref, p string)
"syscall/js.valueDelete": (sp) => {
Reflect.deleteProperty(loadValue(sp + 8), loadString(sp + 16));
},
// func valueIndex(v ref, i int) ref
"syscall/js.valueIndex": (sp) => {
storeValue(sp + 24, Reflect.get(loadValue(sp + 8), getInt64(sp + 16)));
},
// valueSetIndex(v ref, i int, x ref)
"syscall/js.valueSetIndex": (sp) => {
Reflect.set(loadValue(sp + 8), getInt64(sp + 16), loadValue(sp + 24));
},
// func valueCall(v ref, m string, args []ref) (ref, bool)
"syscall/js.valueCall": (sp) => {
try {
const v = loadValue(sp + 8);
const m = Reflect.get(v, loadString(sp + 16));
const args = loadSliceOfValues(sp + 32);
const result = Reflect.apply(m, v, args);
sp = this._inst.exports.getsp(); // see comment above
storeValue(sp + 56, result);
this.mem.setUint8(sp + 64, 1);
} catch (err) {
storeValue(sp + 56, err);
this.mem.setUint8(sp + 64, 0);
}
},
// func valueInvoke(v ref, args []ref) (ref, bool)
"syscall/js.valueInvoke": (sp) => {
try {
const v = loadValue(sp + 8);
const args = loadSliceOfValues(sp + 16);
const result = Reflect.apply(v, undefined, args);
sp = this._inst.exports.getsp(); // see comment above
storeValue(sp + 40, result);
this.mem.setUint8(sp + 48, 1);
} catch (err) {
storeValue(sp + 40, err);
this.mem.setUint8(sp + 48, 0);
}
},
// func valueNew(v ref, args []ref) (ref, bool)
"syscall/js.valueNew": (sp) => {
try {
const v = loadValue(sp + 8);
const args = loadSliceOfValues(sp + 16);
const result = Reflect.construct(v, args);
sp = this._inst.exports.getsp(); // see comment above
storeValue(sp + 40, result);
this.mem.setUint8(sp + 48, 1);
} catch (err) {
storeValue(sp + 40, err);
this.mem.setUint8(sp + 48, 0);
}
},
// func valueLength(v ref) int
"syscall/js.valueLength": (sp) => {
setInt64(sp + 16, parseInt(loadValue(sp + 8).length));
},
// valuePrepareString(v ref) (ref, int)
"syscall/js.valuePrepareString": (sp) => {
const str = encoder.encode(String(loadValue(sp + 8)));
storeValue(sp + 16, str);
setInt64(sp + 24, str.length);
},
// valueLoadString(v ref, b []byte)
"syscall/js.valueLoadString": (sp) => {
const str = loadValue(sp + 8);
loadSlice(sp + 16).set(str);
},
// func valueInstanceOf(v ref, t ref) bool
"syscall/js.valueInstanceOf": (sp) => {
this.mem.setUint8(sp + 24, (loadValue(sp + 8) instanceof loadValue(sp + 16)) ? 1 : 0);
},
// func copyBytesToGo(dst []byte, src ref) (int, bool)
"syscall/js.copyBytesToGo": (sp) => {
const dst = loadSlice(sp + 8);
const src = loadValue(sp + 32);
if (!(src instanceof Uint8Array || src instanceof Uint8ClampedArray)) {
this.mem.setUint8(sp + 48, 0);
return;
}
const toCopy = src.subarray(0, dst.length);
dst.set(toCopy);
setInt64(sp + 40, toCopy.length);
this.mem.setUint8(sp + 48, 1);
},
// func copyBytesToJS(dst ref, src []byte) (int, bool)
"syscall/js.copyBytesToJS": (sp) => {
const dst = loadValue(sp + 8);
const src = loadSlice(sp + 16);
if (!(dst instanceof Uint8Array || dst instanceof Uint8ClampedArray)) {
this.mem.setUint8(sp + 48, 0);
return;
}
const toCopy = src.subarray(0, dst.length);
dst.set(toCopy);
setInt64(sp + 40, toCopy.length);
this.mem.setUint8(sp + 48, 1);
},
"debug": (value) => {
console.log(value);
},
}
};
}
async run(instance) {
this._inst = instance;
this.mem = new DataView(this._inst.exports.mem.buffer);
this._values = [ // JS values that Go currently has references to, indexed by reference id
NaN,
0,
null,
true,
false,
global,
this,
];
this._goRefCounts = new Array(this._values.length).fill(Infinity); // number of references that Go has to a JS value, indexed by reference id
this._ids = new Map([ // mapping from JS values to reference ids
[0, 1],
[null, 2],
[true, 3],
[false, 4],
[global, 5],
[this, 6],
]);
this._idPool = []; // unused ids that have been garbage collected
this.exited = false; // whether the Go program has exited
// Pass command line arguments and environment variables to WebAssembly by writing them to the linear memory.
let offset = 4096;
const strPtr = (str) => {
const ptr = offset;
const bytes = encoder.encode(str + "\0");
new Uint8Array(this.mem.buffer, offset, bytes.length).set(bytes);
offset += bytes.length;
if (offset % 8 !== 0) {
offset += 8 - (offset % 8);
}
return ptr;
};
const argc = this.argv.length;
const argvPtrs = [];
this.argv.forEach((arg) => {
argvPtrs.push(strPtr(arg));
});
argvPtrs.push(0);
const keys = Object.keys(this.env).sort();
keys.forEach((key) => {
argvPtrs.push(strPtr(`${key}=${this.env[key]}`));
});
argvPtrs.push(0);
const argv = offset;
argvPtrs.forEach((ptr) => {
this.mem.setUint32(offset, ptr, true);
this.mem.setUint32(offset + 4, 0, true);
offset += 8;
});
this._inst.exports.run(argc, argv);
if (this.exited) {
this._resolveExitPromise();
}
await this._exitPromise;
}
_resume() {
if (this.exited) {
throw new Error("Go program has already exited");
}
this._inst.exports.resume();
if (this.exited) {
this._resolveExitPromise();
}
}
_makeFuncWrapper(id) {
const go = this;
return function () {
const event = { id: id, this: this, args: arguments };
go._pendingEvent = event;
go._resume();
return event.result;
};
}
}
if (
global.require &&
global.require.main === module &&
global.process &&
global.process.versions &&
!global.process.versions.electron
) {
if (process.argv.length < 3) {
console.error("usage: go_js_wasm_exec [wasm binary] [arguments]");
process.exit(1);
}
const go = new Go();
go.argv = process.argv.slice(2);
go.env = Object.assign({ TMPDIR: require("os").tmpdir() }, process.env);
go.exit = process.exit;
WebAssembly.instantiate(fs.readFileSync(process.argv[2]), go.importObject).then((result) => {
process.on("exit", (code) => { // Node.js exits if no event handler is pending
if (code === 0 && !go.exited) {
// deadlock, make Go print error and stack traces
go._pendingEvent = { id: 0 };
go._resume();
}
});
return go.run(result.instance);
}).catch((err) => {
console.error(err);
process.exit(1);
});
}
})();