From 3802e71772182daebe58cf04c452b6f0b6810a7a Mon Sep 17 00:00:00 2001 From: KernelDeimos Date: Wed, 4 Sep 2024 23:57:45 -0400 Subject: [PATCH 01/18] dev: add emulator image build --- src/emulator/basic.html | 412 ++++++++++++++++++ src/emulator/image/Dockerfile | 57 +++ src/emulator/image/basic-boot | 14 + src/emulator/image/build.sh | 50 +++ src/emulator/image/clean.sh | 3 + src/emulator/image/debug-service | 19 + src/emulator/image/initd/network-service | 17 + src/emulator/image/qemu.sh | 7 + src/emulator/image/rootfs/etc/hostname | 1 + .../image/rootfs/etc/network/interfaces | 4 + src/emulator/image/rootfs/etc/resolv.conf | 1 + src/emulator/image/twisp-service | 25 ++ 12 files changed, 610 insertions(+) create mode 100644 src/emulator/basic.html create mode 100644 src/emulator/image/Dockerfile create mode 100644 src/emulator/image/basic-boot create mode 100755 src/emulator/image/build.sh create mode 100755 src/emulator/image/clean.sh create mode 100644 src/emulator/image/debug-service create mode 100644 src/emulator/image/initd/network-service create mode 100755 src/emulator/image/qemu.sh create mode 100644 src/emulator/image/rootfs/etc/hostname create mode 100644 src/emulator/image/rootfs/etc/network/interfaces create mode 100644 src/emulator/image/rootfs/etc/resolv.conf create mode 100644 src/emulator/image/twisp-service diff --git a/src/emulator/basic.html b/src/emulator/basic.html new file mode 100644 index 0000000000..07bbfb3f26 --- /dev/null +++ b/src/emulator/basic.html @@ -0,0 +1,412 @@ + +Basic Emulator + + + + + + +
+
+ +
diff --git a/src/emulator/image/Dockerfile b/src/emulator/image/Dockerfile new file mode 100644 index 0000000000..c45e3b8efa --- /dev/null +++ b/src/emulator/image/Dockerfile @@ -0,0 +1,57 @@ +FROM i386/alpine:edge + +RUN apk add --update \ + alpine-base bash ncurses shadow curl \ + linux-lts linux-firmware-none linux-headers \ + gcc make gcompat musl-dev libx11-dev xinit \ + bind-tools \ + util-linux \ + htop vim nano \ + && \ + setup-xorg-base xhost xterm xcalc xdotool xkill || true && \ + setup-devd udev || true && \ + touch /root/.Xdefaults && \ + rm /etc/motd /etc/issue && \ + passwd -d root && \ + chsh -s /bin/bash + +RUN apk add neofetch + +COPY basic-boot /etc/init.d/ +RUN chmod +x /etc/init.d/basic-boot + +COPY assets/twisp /bin/twisp +RUN chmod u+x /bin/twisp +COPY twisp-service /etc/init.d/ +RUN chmod +x /etc/init.d/twisp-service +RUN rc-update add twisp-service default + +COPY debug-service /etc/init.d/ +RUN chmod +x /etc/init.d/debug-service +RUN rc-update add debug-service default + +COPY initd/network-service /etc/init.d/ +RUN chmod +x /etc/init.d/network-service +RUN rc-update add network-service default + +# setup init system +# COPY rc.conf /etc/rc.conf +RUN rc-update add dmesg sysinit +RUN rc-update add basic-boot sysinit + +RUN rc-update add root boot +RUN rc-update add localmount boot +RUN rc-update add modules boot +RUN rc-update add sysctl boot +RUN rc-update add bootmisc boot +RUN rc-update add syslog boot + +RUN rc-update add mount-ro shutdown +RUN rc-update add killprocs shutdown +RUN rc-update add savecache shutdown + +COPY rootfs/ / + +RUN setup-hostname puter-alpine + +RUN bash \ No newline at end of file diff --git a/src/emulator/image/basic-boot b/src/emulator/image/basic-boot new file mode 100644 index 0000000000..11e9d3824e --- /dev/null +++ b/src/emulator/image/basic-boot @@ -0,0 +1,14 @@ +#!/sbin/openrc-run + +description="Run Essential Boot Scripts" + +start() { + ebegin "Running Essential Boot Scripts" + mount / -o remount,rw + eend $? +} + +stop() { + ebegin "Stopping Essential Boot Scripts" + eend $? +} diff --git a/src/emulator/image/build.sh b/src/emulator/image/build.sh new file mode 100755 index 0000000000..f8d51fa5e6 --- /dev/null +++ b/src/emulator/image/build.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +set -veu + +if [ -w /var/run/docker.sock ] +then + echo true +else + echo "You aren't in the docker group, please run usermod -a -G docker $USER && newgrp docker" + exit 2 +fi + + +IMAGES="$(dirname "$0")"/build/x86images +OUT_ROOTFS_TAR="$IMAGES"/rootfs.tar +OUT_ROOTFS_BIN="$IMAGES"/rootfs.bin +OUT_ROOTFS_MNT="$IMAGES"/rootfs.mntpoint +CONTAINER_NAME=alpine-full +IMAGE_NAME=i386/alpine-full + +rm -rf $OUT_ROOTFS_BIN || : + +mkdir -p "$IMAGES" +docker build . --platform linux/386 --rm --tag "$IMAGE_NAME" +docker rm "$CONTAINER_NAME" || true +docker create --platform linux/386 -t -i --name "$CONTAINER_NAME" "$IMAGE_NAME" bash + +docker export "$CONTAINER_NAME" > "$OUT_ROOTFS_TAR" +dd if=/dev/zero "of=$OUT_ROOTFS_BIN" bs=512M count=2 + +loop=$(sudo losetup -f) +sudo losetup -P "$loop" "$OUT_ROOTFS_BIN" +sudo mkfs.ext4 "$loop" +mkdir -p "$OUT_ROOTFS_MNT" +sudo mount "$loop" "$OUT_ROOTFS_MNT" + +sudo tar -xf "$OUT_ROOTFS_TAR" -C "$OUT_ROOTFS_MNT" +sudo cp -r "$OUT_ROOTFS_MNT/boot" "$IMAGES/boot" + +sudo umount "$loop" +sudo losetup -d "$loop" +rm "$OUT_ROOTFS_TAR" +rm -rf "$OUT_ROOTFS_MNT" + +echo "done! created" +sudo chown -R $USER:$USER $IMAGES/boot +cd "$IMAGES" +mkdir -p rootfs +split -b50M rootfs.bin rootfs/ +cd ../ +find x86images/rootfs/* | jq -Rnc "[inputs]" diff --git a/src/emulator/image/clean.sh b/src/emulator/image/clean.sh new file mode 100755 index 0000000000..81dec73eb1 --- /dev/null +++ b/src/emulator/image/clean.sh @@ -0,0 +1,3 @@ +sudo umount build/x86images/rootfs.mntpoint +sudo rm -rf ./build + diff --git a/src/emulator/image/debug-service b/src/emulator/image/debug-service new file mode 100644 index 0000000000..d3f48d5a85 --- /dev/null +++ b/src/emulator/image/debug-service @@ -0,0 +1,19 @@ +#!/sbin/openrc-run + +description="Run debug init" + +depend() { + after twisp-service +} + +start() { + ebegin "Running Debug Init" + echo " 🛠 bash will be on tty2" + setsid bash < /dev/tty2 > /dev/tty2 2>&1 & + eend $? +} + +stop() { + ebegin "Stopping Debug Init" + eend $? +} diff --git a/src/emulator/image/initd/network-service b/src/emulator/image/initd/network-service new file mode 100644 index 0000000000..b68ba438ce --- /dev/null +++ b/src/emulator/image/initd/network-service @@ -0,0 +1,17 @@ +#!/sbin/openrc-run + +description="Run network setup" + +start() { + ebegin "Running network setup" + modprobe ne2k-pci + ifupdown ifup eth0 + ip link set lo up + echo "nameserver 192.168.86.1" > /etc/resolv.conf + eend $? +} + +stop() { + ebegin "Stopping network setup" + eend $? +} diff --git a/src/emulator/image/qemu.sh b/src/emulator/image/qemu.sh new file mode 100755 index 0000000000..f0388f53e1 --- /dev/null +++ b/src/emulator/image/qemu.sh @@ -0,0 +1,7 @@ +qemu-system-i386 \ + -kernel ./build/x86images/boot/vmlinuz-lts \ + -initrd ./build/x86images/boot/initramfs-lts \ + -append "rw root=/dev/sda console=ttyS0 init=/sbin/init rootfstype=ext4" \ + -hda ./build/x86images/rootfs.bin \ + -m 1024M \ + -nographic diff --git a/src/emulator/image/rootfs/etc/hostname b/src/emulator/image/rootfs/etc/hostname new file mode 100644 index 0000000000..9e566746c7 --- /dev/null +++ b/src/emulator/image/rootfs/etc/hostname @@ -0,0 +1 @@ +puter-alpine diff --git a/src/emulator/image/rootfs/etc/network/interfaces b/src/emulator/image/rootfs/etc/network/interfaces new file mode 100644 index 0000000000..f6278a2378 --- /dev/null +++ b/src/emulator/image/rootfs/etc/network/interfaces @@ -0,0 +1,4 @@ +iface eth0 inet static + address 192.168.86.100 + netmask 255.255.255.0 + gateway 192.168.86.1 diff --git a/src/emulator/image/rootfs/etc/resolv.conf b/src/emulator/image/rootfs/etc/resolv.conf new file mode 100644 index 0000000000..c43526bf56 --- /dev/null +++ b/src/emulator/image/rootfs/etc/resolv.conf @@ -0,0 +1 @@ +nameserver 192.168.86.1 diff --git a/src/emulator/image/twisp-service b/src/emulator/image/twisp-service new file mode 100644 index 0000000000..8ca743e43f --- /dev/null +++ b/src/emulator/image/twisp-service @@ -0,0 +1,25 @@ +#!/sbin/openrc-run + +description="twisp daemon" +command="/bin/twisp" +command_args="--pty /dev/hvc0" +pidfile="/var/run/twisp.pid" +command_background="yes" +start_stop_daemon_args="--background --make-pidfile" + +depend() { + need localmount + after bootmisc +} + +start() { + ebegin "Starting ${description}" + start-stop-daemon --start --pidfile "${pidfile}" --background --exec ${command} -- ${command_args} + eend $? +} + +stop() { + ebegin "Stopping ${description}" + start-stop-daemon --stop --pidfile "${pidfile}" + eend $? +} From b88aa8731604b8e843879f864e21f3b2662d4b61 Mon Sep 17 00:00:00 2001 From: KernelDeimos Date: Thu, 5 Sep 2024 01:29:26 -0400 Subject: [PATCH 02/18] dev: add twisp submodule --- .gitmodules | 3 +++ submodules/twisp | 1 + 2 files changed, 4 insertions(+) create mode 160000 submodules/twisp diff --git a/.gitmodules b/.gitmodules index 339cb8a471..c2dfcc1d99 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "submodules/v86"] path = submodules/v86 url = git@github.com:copy/v86.git +[submodule "submodules/twisp"] + path = submodules/twisp + url = git@github.com:MercuryWorkshop/twisp.git diff --git a/submodules/twisp b/submodules/twisp new file mode 160000 index 0000000000..ae6e6527d7 --- /dev/null +++ b/submodules/twisp @@ -0,0 +1 @@ +Subproject commit ae6e6527d79c5206c305f27658ff16a6d2840748 From dd8fe8f03e421d922ba051af15991f61183430e7 Mon Sep 17 00:00:00 2001 From: KernelDeimos Date: Thu, 5 Sep 2024 01:30:57 -0400 Subject: [PATCH 03/18] dev: add emulator page --- .../modules/selfhosted/SelfHostedModule.js | 8 + src/emulator/assets/template.html | 19 + src/emulator/image/Dockerfile | 2 - src/emulator/image/assets/.gitignore | 2 + src/emulator/image/build.sh | 7 +- src/emulator/image/build/.gitignore | 2 + src/emulator/src/main.js | 388 +++++++++++++++++- tools/build_v86.sh | 43 ++ 8 files changed, 462 insertions(+), 9 deletions(-) create mode 100644 src/emulator/image/assets/.gitignore create mode 100644 src/emulator/image/build/.gitignore create mode 100755 tools/build_v86.sh diff --git a/src/backend/src/modules/selfhosted/SelfHostedModule.js b/src/backend/src/modules/selfhosted/SelfHostedModule.js index 6ec4b6e2ba..a10989428e 100644 --- a/src/backend/src/modules/selfhosted/SelfHostedModule.js +++ b/src/backend/src/modules/selfhosted/SelfHostedModule.js @@ -117,10 +117,18 @@ class SelfHostedModule extends AdvancedBase { prefix: '/builtin/dev-center', path: path_.resolve(__dirname, RELATIVE_PATH, 'src/dev-center'), }, + { + prefix: '/builtin/emulator/image', + path: path_.resolve(__dirname, RELATIVE_PATH, 'src/emulator/image'), + }, { prefix: '/builtin/emulator', path: path_.resolve(__dirname, RELATIVE_PATH, 'src/emulator/dist'), }, + { + prefix: '/vendor/v86/bios', + path: path_.resolve(__dirname, RELATIVE_PATH, 'submodules/v86/bios'), + }, { prefix: '/vendor/v86', path: path_.resolve(__dirname, RELATIVE_PATH, 'submodules/v86/build'), diff --git a/src/emulator/assets/template.html b/src/emulator/assets/template.html index 87721584de..15d555bce7 100644 --- a/src/emulator/assets/template.html +++ b/src/emulator/assets/template.html @@ -16,6 +16,20 @@ + @@ -41,5 +55,10 @@ <% } %> +
+
+ +
+ \ No newline at end of file diff --git a/src/emulator/image/Dockerfile b/src/emulator/image/Dockerfile index c45e3b8efa..d46fa5974d 100644 --- a/src/emulator/image/Dockerfile +++ b/src/emulator/image/Dockerfile @@ -52,6 +52,4 @@ RUN rc-update add savecache shutdown COPY rootfs/ / -RUN setup-hostname puter-alpine - RUN bash \ No newline at end of file diff --git a/src/emulator/image/assets/.gitignore b/src/emulator/image/assets/.gitignore new file mode 100644 index 0000000000..d6b7ef32c8 --- /dev/null +++ b/src/emulator/image/assets/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/src/emulator/image/build.sh b/src/emulator/image/build.sh index f8d51fa5e6..8e3a45b61c 100755 --- a/src/emulator/image/build.sh +++ b/src/emulator/image/build.sh @@ -10,7 +10,7 @@ else fi -IMAGES="$(dirname "$0")"/build/x86images +IMAGES="$(dirname "$0")"/build OUT_ROOTFS_TAR="$IMAGES"/rootfs.tar OUT_ROOTFS_BIN="$IMAGES"/rootfs.bin OUT_ROOTFS_MNT="$IMAGES"/rootfs.mntpoint @@ -43,8 +43,3 @@ rm -rf "$OUT_ROOTFS_MNT" echo "done! created" sudo chown -R $USER:$USER $IMAGES/boot -cd "$IMAGES" -mkdir -p rootfs -split -b50M rootfs.bin rootfs/ -cd ../ -find x86images/rootfs/* | jq -Rnc "[inputs]" diff --git a/src/emulator/image/build/.gitignore b/src/emulator/image/build/.gitignore new file mode 100644 index 0000000000..d6b7ef32c8 --- /dev/null +++ b/src/emulator/image/build/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/src/emulator/src/main.js b/src/emulator/src/main.js index e4d8630c18..37d5a3c698 100644 --- a/src/emulator/src/main.js +++ b/src/emulator/src/main.js @@ -1 +1,387 @@ -puter.ui.launchApp('editor'); +"use strict"; +// puter.ui.launchApp('editor'); + +// Libs + // SO: 40031688 + function buf2hex(buffer) { // buffer is an ArrayBuffer + return [...new Uint8Array(buffer)] + .map(x => x.toString(16).padStart(2, '0')) + .join(''); + } + +class ATStream { + constructor ({ delegate, acc, transform, observe }) { + this.delegate = delegate; + if ( acc ) this.acc = acc; + if ( transform ) this.transform = transform; + if ( observe ) this.observe = observe; + this.state = {}; + this.carry = []; + } + [Symbol.asyncIterator]() { return this; } + async next_value_ () { + if ( this.carry.length > 0 ) { + console.log('got from carry!', this.carry); + return { + value: this.carry.shift(), + done: false, + }; + } + return await this.delegate.next(); + } + async acc ({ value }) { + return value; + } + async next_ () { + for (;;) { + const ret = await this.next_value_(); + if ( ret.done ) return ret; + const v = await this.acc({ + state: this.state, + value: ret.value, + carry: v => this.carry.push(v), + }); + if ( this.carry.length >= 0 && v === undefined ) { + throw new Error(`no value, but carry value exists`); + } + if ( v === undefined ) continue; + // We have a value, clear the state! + this.state = {}; + if ( this.transform ) { + const new_value = await this.transform( + { value: ret.value }); + return { ...ret, value: new_value }; + } + return { ...ret, value: v }; + } + } + async next () { + const ret = await this.next_(); + if ( this.observe && !ret.done ) { + this.observe(ret); + } + return ret; + } + async enqueue_ (v) { + this.queue.push(v); + } +} + +const NewCallbackByteStream = () => { + let listener; + let queue = []; + const NOOP = () => {}; + let signal = NOOP; + (async () => { + for (;;) { + const v = await new Promise((rslv, rjct) => { + listener = rslv; + }); + queue.push(v); + signal(); + } + })(); + const stream = { + [Symbol.asyncIterator](){ + return this; + }, + async next () { + if ( queue.length > 0 ) { + return { + value: queue.shift(), + done: false, + }; + } + await new Promise(rslv => { + signal = rslv; + }); + signal = NOOP; + const v = queue.shift(); + return { value: v, done: false }; + } + }; + stream.listener = data => { + listener(data); + }; + return stream; +} + +// Tiny inline little-endian integer library +const get_int = (n_bytes, array8, signed=false) => { + return (v => signed ? v : v >>> 0)( + array8.slice(0,n_bytes).reduce((v,e,i)=>v|=e<<8*i,0)); +} +const to_int = (n_bytes, num) => { + return (new Uint8Array()).map((_,i)=>(num>>8*i)&0xFF); +} + +const NewVirtioFrameStream = byteStream => { + return new ATStream({ + delegate: byteStream, + async acc ({ value, carry }) { + if ( ! this.state.buffer ) { + const size = get_int(4, value); + // 512MiB limit in case of attempted abuse or a bug + // (assuming this won't happen under normal conditions) + if ( size > 512*(1024**2) ) { + throw new Error(`Way too much data! (${size} bytes)`); + } + value = value.slice(4); + this.state.buffer = new Uint8Array(size); + this.state.index = 0; + } + + const needed = this.state.buffer.length - this.state.index; + if ( value.length > needed ) { + const remaining = value.slice(needed); + console.log('we got more bytes than we needed', + needed, + remaining, + value.length, + this.state.buffer.length, + this.state.index, + ); + carry(remaining); + } + + const amount = Math.min(value.length, needed); + const added = value.slice(0, amount); + this.state.buffer.set(added, this.state.index); + this.state.index += amount; + + if ( this.state.index > this.state.buffer.length ) { + throw new Error('WUT'); + } + if ( this.state.index == this.state.buffer.length ) { + return this.state.buffer; + } + } + }); +}; + +const wisp_types = [ + { + id: 3, + label: 'CONTINUE', + describe: ({ payload }) => { + return `buffer: ${get_int(4, payload)}B`; + }, + getAttributes ({ payload }) { + return { + buffer_size: get_int(4, payload), + }; + } + }, + { + id: 5, + label: 'INFO', + describe: ({ payload }) => { + return `v${payload[0]}.${payload[1]} ` + + buf2hex(payload.slice(2)); + }, + getAttributes ({ payload }) { + return { + version_major: payload[0], + version_minor: payload[1], + extensions: payload.slice(2), + } + } + }, +]; + +class WispPacket { + static SEND = Symbol('SEND'); + static RECV = Symbol('RECV'); + constructor ({ data, direction, extra }) { + this.direction = direction; + this.data_ = data; + this.extra = extra ?? {}; + this.types_ = { + 1: { label: 'CONNECT' }, + 2: { label: 'DATA' }, + 4: { label: 'CLOSE' }, + }; + for ( const item of wisp_types ) { + this.types_[item.id] = item; + } + } + get type () { + const i_ = this.data_[0]; + return this.types_[i_]; + } + get attributes () { + if ( ! this.type.getAttributes ) return {}; + const attrs = {}; + Object.assign(attrs, this.type.getAttributes({ + payload: this.data_.slice(5), + })); + Object.assign(attrs, this.extra); + return attrs; + } + toVirtioFrame () { + const arry = new Uint8Array(this.data_.length + 4); + arry.set(to_int(4, this.data_.length), 0); + arry.set(this.data_, 4); + return arry; + } + describe () { + return this.type.label + '(' + + (this.type.describe?.({ + payload: this.data_.slice(5), + }) ?? '?') + ')'; + } + log () { + const arrow = + this.direction === this.constructor.SEND ? '->' : + this.direction === this.constructor.RECV ? '<-' : + '<>' ; + console.groupCollapsed(`WISP ${arrow} ${this.describe()}`); + const attrs = this.attributes; + for ( const k in attrs ) { + console.log(k, attrs[k]); + } + console.groupEnd(); + } + reflect () { + const reflected = new WispPacket({ + data: this.data_, + direction: + this.direction === this.constructor.SEND ? + this.constructor.RECV : + this.direction === this.constructor.RECV ? + this.constructor.SEND : + undefined, + extra: { + reflectedFrom: this, + } + }); + return reflected; + } +} + +for ( const item of wisp_types ) { + WispPacket[item.label] = item; +} + +const NewWispPacketStream = frameStream => { + return new ATStream({ + delegate: frameStream, + transform ({ value }) { + return new WispPacket({ + data: value, + direction: WispPacket.RECV, + }); + }, + observe ({ value }) { + value.log(); + } + }); +} + +class WispClient { + constructor ({ + packetStream, + sendFn, + }) { + this.packetStream = packetStream; + this.sendFn = sendFn; + } + send (packet) { + packet.log(); + this.sendFn(packet); + } +} + +window.onload = async function() +{ + const resp = await fetch( + './image/build/rootfs.bin' + ); + const arrayBuffer = await resp.arrayBuffer(); + var emulator = window.emulator = new V86({ + wasm_path: "/vendor/v86/v86.wasm", + memory_size: 512 * 1024 * 1024, + vga_memory_size: 2 * 1024 * 1024, + screen_container: document.getElementById("screen_container"), + bios: { + url: "/vendor/v86/bios/seabios.bin", + }, + vga_bios: { + url: "/vendor/v86/bios/vgabios.bin", + }, + + initrd: { + url: './image/build/boot/initramfs-lts', + }, + bzimage: { + url: './image/build/boot/vmlinuz-lts', + async: false + }, + cmdline: 'rw root=/dev/sda init=/sbin/init rootfstype=ext4', + // cmdline: 'rw root=/dev/sda init=/bin/bash rootfstype=ext4', + // cmdline: "rw init=/sbin/init root=/dev/sda rootfstype=ext4", + // cmdline: "rw init=/sbin/init root=/dev/sda rootfstype=ext4 random.trust_cpu=on 8250.nr_uarts=10 spectre_v2=off pti=off mitigations=off", + + // cdrom: { + // // url: "../images/al32-2024.07.10.iso", + // url: "./image/build/rootfs.bin", + // }, + hda: { + buffer: arrayBuffer, + // url: './image/build/rootfs.bin', + async: true, + // size: 1073741824, + // size: 805306368, + }, + // bzimage_initrd_from_filesystem: true, + autostart: true, + + network_relay_url: "wisp://127.0.0.1:3000", + virtio_console: true, + }); + + + const decoder = new TextDecoder(); + const byteStream = NewCallbackByteStream(); + emulator.add_listener('virtio-console0-output-bytes', + byteStream.listener); + const virtioStream = NewVirtioFrameStream(byteStream); + const wispStream = NewWispPacketStream(virtioStream); + + class PTYManager { + constructor ({ client }) { + this.client = client; + } + init () { + this.run_(); + } + async run_ () { + const handlers_ = { + [WispPacket.INFO.id]: ({ packet }) => { + // console.log('guess we doing info packets now', packet); + this.client.send(packet.reflect()); + } + }; + for await ( const packet of this.client.packetStream ) { + // console.log('what we got here?', + // packet.type, + // packet, + // ); + handlers_[packet.type.id]?.({ packet }); + } + } + } + + const ptyMgr = new PTYManager({ + client: new WispClient({ + packetStream: wispStream, + sendFn: packet => { + emulator.bus.send( + "virtio-console0-input-bytes", + packet.toVirtioFrame(), + ); + } + }) + }); + ptyMgr.init(); +} diff --git a/tools/build_v86.sh b/tools/build_v86.sh new file mode 100755 index 0000000000..d4746225c4 --- /dev/null +++ b/tools/build_v86.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +start_dir=$(pwd) +cleanup() { + cd "$start_dir" +} +trap cleanup ERR EXIT +set -e + +echo -e "\x1B[36;1m<<< Adding Targets >>>\x1B[0m" + +rustup target add wasm32-unknown-unknown +rustup target add i686-unknown-linux-gnu + +echo -e "\x1B[36;1m<<< Building v86 >>>\x1B[0m" + +cd submodules/v86 +make all +cd - + +echo -e "\x1B[36;1m<<< Building Twisp >>>\x1B[0m" + +cd submodules/twisp + +RUSTFLAGS="-C target-feature=+crt-static" cargo build \ + --release \ + --target i686-unknown-linux-gnu \ + `# TODO: what are default features?` \ + --no-default-features + +echo -e "\x1B[36;1m<<< Preparing to Build Imag >>>\x1B[0m" + +cd - + +cp submodules/twisp/target/i686-unknown-linux-gnu/release/twisp \ + src/emulator/image/assets/ + +echo -e "\x1B[36;1m<<< Building Image >>>\x1B[0m" + +cd src/emulator/image +./clean.sh +./build.sh +cd - From cc6790c7f9d08928fdbb290c994f38ba50b1c7b2 Mon Sep 17 00:00:00 2001 From: KernelDeimos Date: Thu, 5 Sep 2024 18:51:06 -0400 Subject: [PATCH 04/18] dev: get basic PTY integration working --- src/emulator/assets/template.html | 10 + src/emulator/src/main.js | 415 +++++++++------------------- src/phoenix/src/pty/XDocumentPTT.js | 8 +- src/puter-wisp/src/exports.js | 106 ++++++- 4 files changed, 243 insertions(+), 296 deletions(-) diff --git a/src/emulator/assets/template.html b/src/emulator/assets/template.html index 15d555bce7..750a05491c 100644 --- a/src/emulator/assets/template.html +++ b/src/emulator/assets/template.html @@ -22,12 +22,22 @@ line-height: 16px; } BODY { + padding: 0; + margin: 0; background-color: #111; display: flex; justify-content: center; align-items: center; height: 100vh; overflow: hidden; + background: linear-gradient(135deg, #232323 50%, transparent 50%) 0% 0% / 3em 3em #101010; + background-position: center center; + background-size: 5px 5px; + } + #screen_container { + padding: 5px; + background-color: #000; + box-shadow: 0 0 32px 0 rgba(0,0,0,0.7); } diff --git a/src/emulator/src/main.js b/src/emulator/src/main.js index 37d5a3c698..6ef3b3dc37 100644 --- a/src/emulator/src/main.js +++ b/src/emulator/src/main.js @@ -1,282 +1,13 @@ "use strict"; -// puter.ui.launchApp('editor'); -// Libs - // SO: 40031688 - function buf2hex(buffer) { // buffer is an ArrayBuffer - return [...new Uint8Array(buffer)] - .map(x => x.toString(16).padStart(2, '0')) - .join(''); - } - -class ATStream { - constructor ({ delegate, acc, transform, observe }) { - this.delegate = delegate; - if ( acc ) this.acc = acc; - if ( transform ) this.transform = transform; - if ( observe ) this.observe = observe; - this.state = {}; - this.carry = []; - } - [Symbol.asyncIterator]() { return this; } - async next_value_ () { - if ( this.carry.length > 0 ) { - console.log('got from carry!', this.carry); - return { - value: this.carry.shift(), - done: false, - }; - } - return await this.delegate.next(); - } - async acc ({ value }) { - return value; - } - async next_ () { - for (;;) { - const ret = await this.next_value_(); - if ( ret.done ) return ret; - const v = await this.acc({ - state: this.state, - value: ret.value, - carry: v => this.carry.push(v), - }); - if ( this.carry.length >= 0 && v === undefined ) { - throw new Error(`no value, but carry value exists`); - } - if ( v === undefined ) continue; - // We have a value, clear the state! - this.state = {}; - if ( this.transform ) { - const new_value = await this.transform( - { value: ret.value }); - return { ...ret, value: new_value }; - } - return { ...ret, value: v }; - } - } - async next () { - const ret = await this.next_(); - if ( this.observe && !ret.done ) { - this.observe(ret); - } - return ret; - } - async enqueue_ (v) { - this.queue.push(v); - } -} - -const NewCallbackByteStream = () => { - let listener; - let queue = []; - const NOOP = () => {}; - let signal = NOOP; - (async () => { - for (;;) { - const v = await new Promise((rslv, rjct) => { - listener = rslv; - }); - queue.push(v); - signal(); - } - })(); - const stream = { - [Symbol.asyncIterator](){ - return this; - }, - async next () { - if ( queue.length > 0 ) { - return { - value: queue.shift(), - done: false, - }; - } - await new Promise(rslv => { - signal = rslv; - }); - signal = NOOP; - const v = queue.shift(); - return { value: v, done: false }; - } - }; - stream.listener = data => { - listener(data); - }; - return stream; -} - -// Tiny inline little-endian integer library -const get_int = (n_bytes, array8, signed=false) => { - return (v => signed ? v : v >>> 0)( - array8.slice(0,n_bytes).reduce((v,e,i)=>v|=e<<8*i,0)); -} -const to_int = (n_bytes, num) => { - return (new Uint8Array()).map((_,i)=>(num>>8*i)&0xFF); -} - -const NewVirtioFrameStream = byteStream => { - return new ATStream({ - delegate: byteStream, - async acc ({ value, carry }) { - if ( ! this.state.buffer ) { - const size = get_int(4, value); - // 512MiB limit in case of attempted abuse or a bug - // (assuming this won't happen under normal conditions) - if ( size > 512*(1024**2) ) { - throw new Error(`Way too much data! (${size} bytes)`); - } - value = value.slice(4); - this.state.buffer = new Uint8Array(size); - this.state.index = 0; - } - - const needed = this.state.buffer.length - this.state.index; - if ( value.length > needed ) { - const remaining = value.slice(needed); - console.log('we got more bytes than we needed', - needed, - remaining, - value.length, - this.state.buffer.length, - this.state.index, - ); - carry(remaining); - } - - const amount = Math.min(value.length, needed); - const added = value.slice(0, amount); - this.state.buffer.set(added, this.state.index); - this.state.index += amount; - - if ( this.state.index > this.state.buffer.length ) { - throw new Error('WUT'); - } - if ( this.state.index == this.state.buffer.length ) { - return this.state.buffer; - } - } - }); -}; - -const wisp_types = [ - { - id: 3, - label: 'CONTINUE', - describe: ({ payload }) => { - return `buffer: ${get_int(4, payload)}B`; - }, - getAttributes ({ payload }) { - return { - buffer_size: get_int(4, payload), - }; - } - }, - { - id: 5, - label: 'INFO', - describe: ({ payload }) => { - return `v${payload[0]}.${payload[1]} ` + - buf2hex(payload.slice(2)); - }, - getAttributes ({ payload }) { - return { - version_major: payload[0], - version_minor: payload[1], - extensions: payload.slice(2), - } - } - }, -]; - -class WispPacket { - static SEND = Symbol('SEND'); - static RECV = Symbol('RECV'); - constructor ({ data, direction, extra }) { - this.direction = direction; - this.data_ = data; - this.extra = extra ?? {}; - this.types_ = { - 1: { label: 'CONNECT' }, - 2: { label: 'DATA' }, - 4: { label: 'CLOSE' }, - }; - for ( const item of wisp_types ) { - this.types_[item.id] = item; - } - } - get type () { - const i_ = this.data_[0]; - return this.types_[i_]; - } - get attributes () { - if ( ! this.type.getAttributes ) return {}; - const attrs = {}; - Object.assign(attrs, this.type.getAttributes({ - payload: this.data_.slice(5), - })); - Object.assign(attrs, this.extra); - return attrs; - } - toVirtioFrame () { - const arry = new Uint8Array(this.data_.length + 4); - arry.set(to_int(4, this.data_.length), 0); - arry.set(this.data_, 4); - return arry; - } - describe () { - return this.type.label + '(' + - (this.type.describe?.({ - payload: this.data_.slice(5), - }) ?? '?') + ')'; - } - log () { - const arrow = - this.direction === this.constructor.SEND ? '->' : - this.direction === this.constructor.RECV ? '<-' : - '<>' ; - console.groupCollapsed(`WISP ${arrow} ${this.describe()}`); - const attrs = this.attributes; - for ( const k in attrs ) { - console.log(k, attrs[k]); - } - console.groupEnd(); - } - reflect () { - const reflected = new WispPacket({ - data: this.data_, - direction: - this.direction === this.constructor.SEND ? - this.constructor.RECV : - this.direction === this.constructor.RECV ? - this.constructor.SEND : - undefined, - extra: { - reflectedFrom: this, - } - }); - return reflected; - } -} - -for ( const item of wisp_types ) { - WispPacket[item.label] = item; -} - -const NewWispPacketStream = frameStream => { - return new ATStream({ - delegate: frameStream, - transform ({ value }) { - return new WispPacket({ - data: value, - direction: WispPacket.RECV, - }); - }, - observe ({ value }) { - value.log(); - } - }); -} +const { XDocumentPTT } = require("../../phoenix/src/pty/XDocumentPTT"); +const { + NewWispPacketStream, + WispPacket, + NewCallbackByteStream, + NewVirtioFrameStream, + DataBuilder, +} = require("../../puter-wisp/src/exports"); class WispClient { constructor ({ @@ -347,28 +78,132 @@ window.onload = async function() byteStream.listener); const virtioStream = NewVirtioFrameStream(byteStream); const wispStream = NewWispPacketStream(virtioStream); + + const shell = puter.ui.parentApp(); + const ptt = new XDocumentPTT(shell, { + disableReader: true, + }) + + ptt.termios.echo = false; class PTYManager { + static STATE_INIT = { + name: 'init', + handlers: { + [WispPacket.INFO.id]: function ({ packet }) { + this.client.send(packet.reflect()); + this.state = this.constructor.STATE_READY; + } + } + }; + static STATE_READY = { + name: 'ready', + handlers: { + [WispPacket.DATA.id]: function ({ packet }) { + console.log('stream id?', packet.streamId); + const pty = this.stream_listeners_[packet.streamId]; + pty.on_payload(packet.payload); + } + }, + on: function () { + const pty = this.getPTY(); + console.log('PTY created', pty); + pty.on_payload = data => { + ptt.out.write(data); + } + (async () => { + // for (;;) { + // const buff = await ptt.in.read(); + // if ( buff === undefined ) continue; + // console.log('this is what ptt in gave', buff); + // pty.send(buff); + // } + const stream = ptt.readableStream; + for await ( const chunk of stream ) { + if ( chunk === undefined ) { + console.error('huh, missing chunk', chunk); + continue; + } + pty.send(chunk); + } + })() + }, + } + + set state (value) { + console.log('[PTYManager] State updated: ', value.name); + this.state_ = value; + if ( this.state_.on ) { + this.state_.on.call(this) + } + } + get state () { return this.state_ } + constructor ({ client }) { + this.streamId = 0; + this.state_ = null; this.client = client; + this.state = this.constructor.STATE_INIT; + this.stream_listeners_ = {}; } init () { this.run_(); } async run_ () { - const handlers_ = { - [WispPacket.INFO.id]: ({ packet }) => { - // console.log('guess we doing info packets now', packet); - this.client.send(packet.reflect()); - } - }; for await ( const packet of this.client.packetStream ) { - // console.log('what we got here?', - // packet.type, - // packet, - // ); - handlers_[packet.type.id]?.({ packet }); + const handlers_ = this.state_.handlers; + if ( ! handlers_[packet.type.id] ) { + console.error(`No handler for packet type ${packet.type.id}`); + console.log(handlers_, this); + continue; + } + handlers_[packet.type.id].call(this, { packet }); + } + } + + getPTY () { + const streamId = ++this.streamId; + const data = new DataBuilder({ leb: true }) + .uint8(0x01) + .uint32(streamId) + .uint8(0x03) + .uint16(10) + .utf8('/bin/bash') + // .utf8('/usr/bin/htop') + .build(); + const packet = new WispPacket( + { data, direction: WispPacket.SEND }); + this.client.send(packet); + const pty = new PTY({ client: this.client, streamId }); + console.log('setting to stream id', streamId); + this.stream_listeners_[streamId] = pty; + return pty; + } + } + + class PTY { + constructor ({ client, streamId }) { + this.client = client; + this.streamId = streamId; + } + + on_payload (data) { + + } + + send (data) { + // convert text into buffers + if ( typeof data === 'string' ) { + data = (new TextEncoder()).encode(data, 'utf-8') } + data = new DataBuilder({ leb: true }) + .uint8(0x02) + .uint32(this.streamId) + .cat(data) + .build(); + const packet = new WispPacket( + { data, direction: WispPacket.SEND }); + this.client.send(packet); } } @@ -376,9 +211,11 @@ window.onload = async function() client: new WispClient({ packetStream: wispStream, sendFn: packet => { + const virtioframe = packet.toVirtioFrame(); + console.log('virtio frame', virtioframe); emulator.bus.send( "virtio-console0-input-bytes", - packet.toVirtioFrame(), + virtioframe, ); } }) diff --git a/src/phoenix/src/pty/XDocumentPTT.js b/src/phoenix/src/pty/XDocumentPTT.js index b1054d3962..7cb6a9bb83 100644 --- a/src/phoenix/src/pty/XDocumentPTT.js +++ b/src/phoenix/src/pty/XDocumentPTT.js @@ -26,7 +26,7 @@ export class XDocumentPTT { id: 104, }, } - constructor(terminalConnection) { + constructor(terminalConnection, opts = {}) { for ( const k in XDocumentPTT.IOCTL ) { this[k] = async () => { return await new Promise((resolve, reject) => { @@ -75,8 +75,10 @@ export class XDocumentPTT { } }); this.out = this.writableStream.getWriter(); - this.in = this.readableStream.getReader(); - this.in = new BetterReader({ delegate: this.in }); + if ( ! opts.disableReader ) { + this.in = this.readableStream.getReader(); + this.in = new BetterReader({ delegate: this.in }); + } terminalConnection.on('message', message => { if (message.$ === 'ioctl.set') { diff --git a/src/puter-wisp/src/exports.js b/src/puter-wisp/src/exports.js index 709bb3eba6..8d9d8eeb06 100644 --- a/src/puter-wisp/src/exports.js +++ b/src/puter-wisp/src/exports.js @@ -13,7 +13,7 @@ lib.get_int = (n_bytes, array8, signed=false) => { array8.slice(0,n_bytes).reduce((v,e,i)=>v|=e<<8*i,0)); } lib.to_int = (n_bytes, num) => { - return (new Uint8Array()).map((_,i)=>(num>>8*i)&0xFF); + return (new Uint8Array(n_bytes)).map((_,i)=>(num>>8*i)&0xFF); } // Accumulator and/or Transformer (and/or Observer) Stream @@ -183,6 +183,26 @@ const wisp_types = [ }; } }, + { + id: 1, + label: 'CONNECT', + describe: ({ attributes }) => { + return `${ + attributes.type === 1 ? 'TCP' : + attributes.type === 2 ? 'UDP' : + attributes.type === 3 ? 'PTY' : + 'UNKNOWN' + } ${attributes.host}:${attributes.port}`; + }, + getAttributes: ({ payload }) => { + const type = payload[0]; + const port = lib.get_int(2, payload.slice(1)); + const host = new TextDecoder().decode(payload.slice(3)); + return { + type, port, host, + }; + } + }, { id: 5, label: 'INFO', @@ -198,6 +218,20 @@ const wisp_types = [ } } }, + { + id: 2, + label: 'DATA', + describe: ({ attributes }) => { + return `${attributes.length}B`; + }, + getAttributes ({ payload }) { + return { + length: payload.length, + contents: payload, + utf8: new TextDecoder().decode(payload), + } + } + }, ]; class WispPacket { @@ -208,8 +242,6 @@ class WispPacket { this.data_ = data; this.extra = extra ?? {}; this.types_ = { - 1: { label: 'CONNECT' }, - 2: { label: 'DATA' }, 4: { label: 'CLOSE' }, }; for ( const item of wisp_types ) { @@ -222,14 +254,28 @@ class WispPacket { } get attributes () { if ( ! this.type.getAttributes ) return {}; - const attrs = {}; + const attrs = { + streamId: this.streamId, + }; Object.assign(attrs, this.type.getAttributes({ payload: this.data_.slice(5), })); Object.assign(attrs, this.extra); return attrs; } + get payload () { + return this.data_.slice(5); + } + get streamId () { + return lib.get_int(4, this.data_.slice(1)); + } toVirtioFrame () { + console.log( + 'WISP packet to virtio frame', + this.data_, + this.data_.length, + lib.to_int(4, this.data_.length), + ); const arry = new Uint8Array(this.data_.length + 4); arry.set(lib.to_int(4, this.data_.length), 0); arry.set(this.data_, 4); @@ -238,6 +284,7 @@ class WispPacket { describe () { return this.type.label + '(' + (this.type.describe?.({ + attributes: this.attributes, payload: this.data_.slice(5), }) ?? '?') + ')'; } @@ -290,9 +337,60 @@ const NewWispPacketStream = frameStream => { }); } +class DataBuilder { + constructor ({ leb } = {}) { + this.pos = 0; + this.steps = []; + this.leb = leb; + } + uint8(value) { + this.steps.push(['setUint8', this.pos, value]); + this.pos++; + return this; + } + uint16(value, leb) { + leb ??= this.leb; + this.steps.push(['setUint8', this.pos, value, leb]); + this.pos += 2; + return this; + } + uint32(value, leb) { + leb ??= this.leb; + this.steps.push(['setUint32', this.pos, value, leb]); + this.pos += 4; + return this; + } + utf8(value) { + const encoded = new TextEncoder().encode(value); + this.steps.push(['array', 'set', encoded, this.pos]); + this.pos += encoded.length; + return this; + } + cat(data) { + this.steps.push(['array', 'set', data, this.pos]); + this.pos += data.length; + return this; + } + build () { + const array = new Uint8Array(this.pos); + const view = new DataView(array.buffer); + for ( const step of this.steps ) { + let target = view; + let fn_name = step.shift(); + if ( fn_name === 'array' ) { + fn_name = step.shift(); + target = array; + } + target[fn_name](...step); + } + return array; + } +} + module.exports = { NewCallbackByteStream, NewVirtioFrameStream, NewWispPacketStream, WispPacket, + DataBuilder, }; From 68a5b4e4b0a8bcd84318b68f3672e9377c9e8754 Mon Sep 17 00:00:00 2001 From: KernelDeimos Date: Sun, 8 Sep 2024 01:44:13 -0400 Subject: [PATCH 05/18] fix: simplify callback listener and fix async bug --- src/puter-wisp/src/exports.js | 13 +--- src/puter-wisp/test/test.js | 128 ++++++++++++++++++++++++++++++---- 2 files changed, 115 insertions(+), 26 deletions(-) diff --git a/src/puter-wisp/src/exports.js b/src/puter-wisp/src/exports.js index 8d9d8eeb06..39f06e0ddd 100644 --- a/src/puter-wisp/src/exports.js +++ b/src/puter-wisp/src/exports.js @@ -77,19 +77,9 @@ class ATStream { } const NewCallbackByteStream = () => { - let listener; let queue = []; const NOOP = () => {}; let signal = NOOP; - (async () => { - for (;;) { - const v = await new Promise((rslv, rjct) => { - listener = rslv; - }); - queue.push(v); - signal(); - } - })(); const stream = { [Symbol.asyncIterator](){ return this; @@ -110,7 +100,8 @@ const NewCallbackByteStream = () => { } }; stream.listener = data => { - listener(data); + queue.push(data); + signal(); }; return stream; } diff --git a/src/puter-wisp/test/test.js b/src/puter-wisp/test/test.js index be063cee0d..53f5b54c7d 100644 --- a/src/puter-wisp/test/test.js +++ b/src/puter-wisp/test/test.js @@ -19,32 +19,130 @@ const NewTestFullByteStream = uint8array => { })(); }; -(async () => { +/** + * This will send 'sz'-sized chunks of the uint8array + * until the uint8array is exhausted. The last chunk + * may be smaller than 'sz'. + * @curry + * @param {*} sz + * @param {*} uint8array + */ +const NewTestWindowByteStream = sz => { + const fn = uint8array => { + return (async function * () { + let offset = 0; + while ( offset < uint8array.length ) { + const end = Math.min(offset + sz, uint8array.length); + const chunk = uint8array.slice(offset, end); + offset += sz; + yield chunk; + } + })(); + }; + fn.name_ = `NewTestWindowByteStream(${sz})`; + return fn; +}; + +const NewTestChunkedByteStream = chunks => { + return (async function * () { + for ( const chunk of chunks ) { + yield chunk; + } + })(); +} + +const test = async (name, fn) => { + console.log(`\x1B[36;1m=== [ Running test: ${name} ] ===\x1B[0m`); + await fn(); +}; + +const BASH_TEST_BYTES = [ + 22, 0, 0, 0, 2, 1, 0, 0, 0, 27, 91, 63, 50, 48, 48, 52, 108, 13, 27, 91, 63, 50, 48, 48, 52, 104, + 10, 0, 0, 0, 2, 1, 0, 0, 0, 40, 110, 111, 110, 101, + 10, 0, 0, 0, 2, 1, 0, 0, 0, 41, 58, 47, 35, 32, + 7, 0, 0, 0, 2, 1, 0, 0, 0, 13, 10, + 14, 0, 0, 0, 2, 1, 0, 0, 0, 27, 91, 63, 50, 48, 48, 52, 108, 13, + 17, 0, 0, 0, 2, 1, 0, 0, 0, 27, 91, 63, 50, 48, 48, 52, 104, 40, 110, 111, 110, + 11, 0, 0, 0, 2, 1, 0, 0, 0, 101, 41, 58, 47, 35, 32 +] + +const runit = async () => { const stream_behaviors = [ NewTestByteStream, NewTestFullByteStream, + NewTestWindowByteStream(2), + NewTestWindowByteStream(3), ]; + for ( const stream_behavior of stream_behaviors ) { - const byteStream = stream_behavior( + await test(`Wisp CONTINUE ${stream_behavior.name_ ?? stream_behavior.name}`, async () => { + const byteStream = stream_behavior( + Uint8Array.from([ + 9, 0, 0, 0, // size of frame: 9 bytes (u32-L) + 3, // CONTINUE (u8) + 0, 0, 0, 0, // stream id: 0 (u32-L) + 0x0F, 0x0F, 0, 0, // buffer size (u32-L) + ]) + ); + const virtioStream = NewVirtioFrameStream(byteStream); + const wispStream = NewWispPacketStream(virtioStream); + + const packets = []; + for await ( const packet of wispStream ) { + packets.push(packet); + } + + assert.strictEqual(packets.length, 1); + const packet = packets[0]; + assert.strictEqual(packet.type.id, 3); + assert.strictEqual(packet.type.label, 'CONTINUE'); + assert.strictEqual(packet.type, WispPacket.CONTINUE); + }); + } + + await test('bash prompt chunking', async () => { + const byteStream = NewTestChunkedByteStream([ + // These are data frames from virtio->twisp->bash + // "(none" + Uint8Array.from([ + 10, 0, 0, 0, 2, 1, 0, 0, 0, + 40, 110, 111, 110, 101 + ]), + // "):/# " Uint8Array.from([ - 9, 0, 0, 0, // size of frame: 9 bytes (u32-L) - 3, // CONTINUE (u8) - 0, 0, 0, 0, // stream id: 0 (u32-L) - 0x0F, 0x0F, 0, 0, // buffer size (u32-L) - ]) - ); + 10, 0, 0, 0, 2, 1, 0, 0, 0, + 41, 58, 47, 35, 32, + ]), + ]); const virtioStream = NewVirtioFrameStream(byteStream); const wispStream = NewWispPacketStream(virtioStream); - const packets = []; + const data = []; for await ( const packet of wispStream ) { - packets.push(packet); + for ( const item of packet.payload ) { + data.push(item); + } } - assert.strictEqual(packets.length, 1); - const packet = packets[0]; - assert.strictEqual(packet.type.id, 3); - assert.strictEqual(packet.type.label, 'CONTINUE'); - assert.strictEqual(packet.type, WispPacket.CONTINUE); + const expected = [ + 40, 110, 111, 110, 101, + 41, 58, 47, 35, 32, + ]; + + assert.strictEqual(data.length, expected.length); + for ( let i = 0; i < data.length; i++ ) { + assert.strictEqual(data[i], expected[i]); + } + }); +}; + +(async () => { + try { + await runit(); + } catch (e) { + console.error(e); + console.log(`\x1B[31;1mTest Failed\x1B[0m`); + process.exit(1); } + console.log(`\x1B[32;1mAll tests passed\x1B[0m`); })(); \ No newline at end of file From 258b3cafd1af535bbac3e3f7d49521a64f88fa45 Mon Sep 17 00:00:00 2001 From: KernelDeimos Date: Sun, 8 Sep 2024 23:52:45 -0400 Subject: [PATCH 06/18] dev: resize handling --- src/emulator/src/main.js | 24 +++++++++++++++++++ src/phoenix/src/ansi-shell/ANSIShell.js | 1 + .../providers/PuterAppCommandProvider.js | 15 +++++++++++- src/puter-wisp/src/exports.js | 14 +++++++++++ 4 files changed, 53 insertions(+), 1 deletion(-) diff --git a/src/emulator/src/main.js b/src/emulator/src/main.js index 6ef3b3dc37..633d0a0dac 100644 --- a/src/emulator/src/main.js +++ b/src/emulator/src/main.js @@ -108,9 +108,20 @@ window.onload = async function() on: function () { const pty = this.getPTY(); console.log('PTY created', pty); + + // resize + ptt.on('ioctl.set', evt => { + console.log('event?', evt); + pty.resize(evt.windowSize); + }); + ptt.TIOCGWINSZ(); + + // data from PTY pty.on_payload = data => { ptt.out.write(data); } + + // data to PTY (async () => { // for (;;) { // const buff = await ptt.in.read(); @@ -205,6 +216,19 @@ window.onload = async function() { data, direction: WispPacket.SEND }); this.client.send(packet); } + + resize ({ rows, cols }) { + console.log('resize called!'); + const data = new DataBuilder({ leb: true }) + .uint8(0xf0) + .uint32(this.streamId) + .uint16(rows) + .uint16(cols) + .build(); + const packet = new WispPacket( + { data, direction: WispPacket.SEND }); + this.client.send(packet); + } } const ptyMgr = new PTYManager({ diff --git a/src/phoenix/src/ansi-shell/ANSIShell.js b/src/phoenix/src/ansi-shell/ANSIShell.js index 580ba2b7c5..4d9cd47f70 100644 --- a/src/phoenix/src/ansi-shell/ANSIShell.js +++ b/src/phoenix/src/ansi-shell/ANSIShell.js @@ -219,6 +219,7 @@ export class ANSIShell extends EventTarget { } const executionCtx = this.ctx.sub({ + shell: this, vars: this.variables, env: this.env, locals: { diff --git a/src/phoenix/src/puter-shell/providers/PuterAppCommandProvider.js b/src/phoenix/src/puter-shell/providers/PuterAppCommandProvider.js index 2014989335..ff88897546 100644 --- a/src/phoenix/src/puter-shell/providers/PuterAppCommandProvider.js +++ b/src/phoenix/src/puter-shell/providers/PuterAppCommandProvider.js @@ -60,6 +60,17 @@ export class PuterAppCommandProvider { }; const child = await puter.ui.launchApp(id, args); + const resize_listener = evt => { + child.postMessage({ + $: 'ioctl.set', + windowSize: { + rows: evt.detail.rows, + cols: evt.detail.cols, + } + }); + }; + ctx.shell.addEventListener('signal.window-resize', resize_listener); + // Wait for app to close. const app_close_promise = new Promise((resolve, reject) => { child.on('close', (data) => { @@ -118,7 +129,9 @@ export class PuterAppCommandProvider { } // TODO: propagate sigint to the app - return Promise.race([ app_close_promise, sigint_promise ]); + const exit = await Promise.race([ app_close_promise, sigint_promise ]); + ctx.shell.removeEventListener('signal.window-resize', resize_listener); + return exit; } }; } diff --git a/src/puter-wisp/src/exports.js b/src/puter-wisp/src/exports.js index 39f06e0ddd..88b5028324 100644 --- a/src/puter-wisp/src/exports.js +++ b/src/puter-wisp/src/exports.js @@ -223,6 +223,20 @@ const wisp_types = [ } } }, + { + // TODO: extension types should not be hardcoded here + id: 0xf0, + label: 'RESIZE', + describe: ({ attributes }) => { + return `${attributes.cols}x${attributes.rows}`; + }, + getAttributes ({ payload }) { + return { + rows: lib.get_int(2, payload), + cols: lib.get_int(2, payload.slice(2)), + } + } + }, ]; class WispPacket { From 0acd904176e31adfc543dfa4ec85b433f572e7d2 Mon Sep 17 00:00:00 2001 From: KernelDeimos Date: Mon, 9 Sep 2024 16:24:56 -0400 Subject: [PATCH 07/18] dev: add startup apps, start emulator by default --- doc/devmeta/track-comments.md | 3 +++ src/backend/src/services/BaseService.js | 4 ++-- src/gui/src/UI/UIDesktop.js | 6 ++++++ src/gui/src/definitions.js | 9 +++++++-- src/gui/src/initgui.js | 5 +++++ src/gui/src/services/ProcessService.js | 17 ++++++++++++++++ src/putility/index.js | 4 ++++ src/putility/src/concepts/Service.js | 26 +++++++++++++++++++++++++ 8 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 src/putility/src/concepts/Service.js diff --git a/doc/devmeta/track-comments.md b/doc/devmeta/track-comments.md index 1c373262e2..260d7f3330 100644 --- a/doc/devmeta/track-comments.md +++ b/doc/devmeta/track-comments.md @@ -57,3 +57,6 @@ Comments beginning with `// track:`. See It may be applicable to write an iterator in the future, or something will come up that require these to be handled with a modular approach instead. +- `track: checkpoint` + A location where some statement about the state of the + software must hold true. diff --git a/src/backend/src/services/BaseService.js b/src/backend/src/services/BaseService.js index 83ba918a8f..1b169fae6d 100644 --- a/src/backend/src/services/BaseService.js +++ b/src/backend/src/services/BaseService.js @@ -16,11 +16,11 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -const { AdvancedBase } = require("../../../putility"); +const { concepts } = require("@heyputer/putility"); const NOOP = async () => {}; -class BaseService extends AdvancedBase { +class BaseService extends concepts.Service { constructor (service_resources, ...a) { const { services, config, my_config, name, args } = service_resources; super(service_resources, ...a); diff --git a/src/gui/src/UI/UIDesktop.js b/src/gui/src/UI/UIDesktop.js index 8a0ea74482..d78170b3c6 100644 --- a/src/gui/src/UI/UIDesktop.js +++ b/src/gui/src/UI/UIDesktop.js @@ -1022,6 +1022,12 @@ async function UIDesktop(options){ // adjust window container to take into account the toolbar height $('.window-container').css('top', window.toolbar_height); + // track: checkpoint + //----------------------------- + // GUI is ready to launch apps! + //----------------------------- + + globalThis.services.emit('gui:ready'); //-------------------------------------------------------------------------------------- // Determine if an app was launched from URL diff --git a/src/gui/src/definitions.js b/src/gui/src/definitions.js index 8e5911ed1e..7f928bca5a 100644 --- a/src/gui/src/definitions.js +++ b/src/gui/src/definitions.js @@ -17,9 +17,13 @@ * along with this program. If not, see . */ -import { AdvancedBase } from "@heyputer/putility"; +import { concepts, AdvancedBase } from "@heyputer/putility"; -export class Service { +export class Service extends concepts.Service { + // TODO: Service todo items + static TODO = [ + 'consolidate with BaseService from backend' + ]; construct (o) { this.$puter = {}; for ( const k in o ) this.$puter[k] = o[k]; @@ -28,6 +32,7 @@ export class Service { } init (...a) { if ( ! this._init ) return; + this.services = a[0].services; return this._init(...a) } }; diff --git a/src/gui/src/initgui.js b/src/gui/src/initgui.js index df19077815..0506994edf 100644 --- a/src/gui/src/initgui.js +++ b/src/gui/src/initgui.js @@ -55,6 +55,11 @@ const launch_services = async function (options) { const services_m_ = {}; globalThis.services = { get: (name) => services_m_[name], + emit: (id, args) => { + for (const [_, instance] of services_l_) { + instance.__on(id, args ?? []); + } + } }; const register = (name, instance) => { services_l_.push([name, instance]); diff --git a/src/gui/src/services/ProcessService.js b/src/gui/src/services/ProcessService.js index 08683772b4..b210a6b6ca 100644 --- a/src/gui/src/services/ProcessService.js +++ b/src/gui/src/services/ProcessService.js @@ -22,6 +22,10 @@ import { InitProcess, Service } from "../definitions.js"; const NULL_UUID = '00000000-0000-0000-0000-000000000000'; export class ProcessService extends Service { + static INITRC = [ + 'test-emu' + ]; + async _init () { this.processes = []; this.processes_map = new Map(); @@ -33,6 +37,19 @@ export class ProcessService extends Service { this.register_(root); } + ['__on_gui:ready'] () { + const svc_exec = this.services.get('exec'); + for ( let spec of ProcessService.INITRC ) { + if ( typeof spec === 'string' ) { + spec = { name: spec }; + } + + svc_exec.launchApp({ + app_name: spec.name, + }); + } + } + get_init () { return this.processes_map.get(NULL_UUID); } diff --git a/src/putility/index.js b/src/putility/index.js index ce7be26ae1..74835bcd5d 100644 --- a/src/putility/index.js +++ b/src/putility/index.js @@ -17,10 +17,14 @@ * along with this program. If not, see . */ const { AdvancedBase } = require('./src/AdvancedBase'); +const { Service } = require('./src/concepts/Service'); module.exports = { AdvancedBase, libs: { promise: require('./src/libs/promise'), }, + concepts: { + Service, + }, }; diff --git a/src/putility/src/concepts/Service.js b/src/putility/src/concepts/Service.js new file mode 100644 index 0000000000..023beb7806 --- /dev/null +++ b/src/putility/src/concepts/Service.js @@ -0,0 +1,26 @@ +const { AdvancedBase } = require("../AdvancedBase"); + +const NOOP = async () => {}; + +/** + * Service will be incrementally updated to consolidate + * BaseService in Puter's backend with Service in Puter's frontend, + * becoming the common base for both and a useful utility in general. + */ +class Service extends AdvancedBase { + async __on (id, args) { + const handler = this.__get_event_handler(id); + + return await handler(id, ...args); + } + + __get_event_handler (id) { + return this[`__on_${id}`]?.bind?.(this) + || this.constructor[`__on_${id}`]?.bind?.(this.constructor) + || NOOP; + } +} + +module.exports = { + Service, +}; From e0a79da74503ef25debd557fa12dedc84b746f27 Mon Sep 17 00:00:00 2001 From: KernelDeimos Date: Mon, 9 Sep 2024 18:25:33 -0400 Subject: [PATCH 08/18] feat: add connectToInstance method to puter.ui --- src/emulator/src/main.js | 4 ++ src/gui/src/definitions.js | 13 ++++++ src/gui/src/services/ExecService.js | 33 +++++++++++++ src/gui/src/services/ProcessService.js | 10 ++++ src/phoenix/src/puter-shell/main.js | 2 + .../providers/EmuCommandProvider.js | 46 +++++++++++++++++++ src/puter-js/src/modules/UI.js | 24 ++++++++++ 7 files changed, 132 insertions(+) create mode 100644 src/phoenix/src/puter-shell/providers/EmuCommandProvider.js diff --git a/src/emulator/src/main.js b/src/emulator/src/main.js index 633d0a0dac..1b7d7f9992 100644 --- a/src/emulator/src/main.js +++ b/src/emulator/src/main.js @@ -23,6 +23,10 @@ class WispClient { } } +puter.ui.on('connection', event => { + console.log('emulator got connection event', event); +}); + window.onload = async function() { const resp = await fetch( diff --git a/src/gui/src/definitions.js b/src/gui/src/definitions.js index 7f928bca5a..5ef2f925c7 100644 --- a/src/gui/src/definitions.js +++ b/src/gui/src/definitions.js @@ -85,6 +85,10 @@ export class Process extends AdvancedBase{ this._signal(sig); } + handle_connection (other_process) { + throw new Error('Not implemented'); + } + get type () { const _to_type_name = (name) => { return name.replace(/Process$/, '').toLowerCase(); @@ -139,6 +143,15 @@ export class PortalProcess extends Process { const target = this.references.iframe.contentWindow; // NEXT: ... } + + handle_connection (other_process, args) { + const target = this.references.iframe.contentWindow; + target.postMessage({ + msg: 'connection', + appInstanceID: other_process.uuid, + args, + }); + } }; export class PseudoProcess extends Process { _construct () { this.type_ = 'ui' } diff --git a/src/gui/src/services/ExecService.js b/src/gui/src/services/ExecService.js index 15ef144cff..213705b911 100644 --- a/src/gui/src/services/ExecService.js +++ b/src/gui/src/services/ExecService.js @@ -11,6 +11,9 @@ export class ExecService extends Service { svc_ipc.register_ipc_handler('launchApp', { handler: this.launchApp.bind(this), }); + svc_ipc.register_ipc_handler('connectToInstance', { + handler: this.connectToInstance.bind(this), + }); } // This method is exposed to apps via IPCService. @@ -70,4 +73,34 @@ export class ExecService extends Service { usesSDK: true, }; } + + async connectToInstance ({ app_name, args }, { ipc_context, msg_id } = {}) { + const caller_process = ipc_context?.caller?.process; + if ( ! caller_process ) { + throw new Error('Caller process not found'); + } + + console.log( + caller_process.name, + app_name, + ); + // TODO: permissions integration; for now it's hardcoded + if ( caller_process.name !== 'phoenix' ) { + throw new Error('Connection not allowed.'); + } + if ( app_name !== 'test-emu' ) { + throw new Error('Connection not allowed.'); + } + + const svc_process = this.services.get('process'); + const options = svc_process.select_by_name(app_name); + const process = options[0]; + + await process.handle_connection(caller_process, args); + + return { + appInstanceID: process.uuid, + response, + }; + } } diff --git a/src/gui/src/services/ProcessService.js b/src/gui/src/services/ProcessService.js index b210a6b6ca..d41404ee3c 100644 --- a/src/gui/src/services/ProcessService.js +++ b/src/gui/src/services/ProcessService.js @@ -66,6 +66,16 @@ export class ProcessService extends Service { return this.uuid_to_treelist.get(uuid); } + select_by_name (name) { + const list = []; + for ( const process of this.processes ) { + if ( process.name === name ) { + list.push(process); + } + } + return list; + } + register (process) { this.register_(process); this.attach_to_parent_(process); diff --git a/src/phoenix/src/puter-shell/main.js b/src/phoenix/src/puter-shell/main.js index 0d4b94cd6a..e79252dad1 100644 --- a/src/phoenix/src/puter-shell/main.js +++ b/src/phoenix/src/puter-shell/main.js @@ -35,6 +35,7 @@ import { MultiWriter } from '../ansi-shell/ioutil/MultiWriter.js'; import { CompositeCommandProvider } from './providers/CompositeCommandProvider.js'; import { ScriptCommandProvider } from './providers/ScriptCommandProvider.js'; import { PuterAppCommandProvider } from './providers/PuterAppCommandProvider.js'; +import { EmuCommandProvider } from './providers/EmuCommandProvider.js'; const argparser_registry = { [SimpleArgParser.name]: SimpleArgParser @@ -92,6 +93,7 @@ export const launchPuterShell = async (ctx) => { // PuterAppCommandProvider is only usable on Puter ...(ctx.platform.name === 'puter' ? [new PuterAppCommandProvider()] : []), new ScriptCommandProvider(), + new EmuCommandProvider(), ]); ctx = ctx.sub({ diff --git a/src/phoenix/src/puter-shell/providers/EmuCommandProvider.js b/src/phoenix/src/puter-shell/providers/EmuCommandProvider.js new file mode 100644 index 0000000000..1af4d9d6ea --- /dev/null +++ b/src/phoenix/src/puter-shell/providers/EmuCommandProvider.js @@ -0,0 +1,46 @@ +import { Exit } from "../coreutils/coreutil_lib/exit"; + +export class EmuCommandProvider { + static AVAILABLE = [ + 'bash', + 'htop', + ]; + + static EMU_APP_NAME = 'test-emu'; + + constructor () { + this.available = this.constructor.AVAILABLE; + this.emulator = null; + } + + async aquire_emulator () { + if ( this.emulator ) return this.emulator; + + // FUTURE: when we have a way to query instances + // without exposing the real instance id + /* + const instances = await puter.ui.queryInstances(); + if ( instances.length < 0 ) { + return; + } + const instance = instances[0]; + */ + + const conn = await puter.ui.connectToInstance(this.constructor.EMU_APP_NAME); + return this.emulator = conn; + } + + async lookup (id, { ctx }) { + if ( ! this.available.includes(id) ) { + return; + } + + const emu = await this.aquire_emulator(); + if ( ! emu ) { + ctx.externs.out.write('No emulator available.\n'); + return new Exit(1); + } + + ctx.externs.out.write(`Launching ${id} in emulator ${emu.appInstanceID}\n`); + } +} diff --git a/src/puter-js/src/modules/UI.js b/src/puter-js/src/modules/UI.js index 2cf327a5c5..f8d9a794a3 100644 --- a/src/puter-js/src/modules/UI.js +++ b/src/puter-js/src/modules/UI.js @@ -203,6 +203,7 @@ class UI extends EventListener { const eventNames = [ 'localeChanged', 'themeChanged', + 'connection', ]; super(eventNames); this.#eventNames = eventNames; @@ -460,6 +461,15 @@ class UI extends EventListener { this.emit(name, data); this.#lastBroadcastValue.set(name, data); } + else if ( e.data.msg === 'connection' ) { + const conn = AppConnection.from(e.data, { + appInstanceID: this.appInstanceID, + messageTarget: window.parent, + }); + this.emit('connection', { + conn + }); + } }); // We need to send the mouse position to the host environment @@ -951,6 +961,20 @@ class UI extends EventListener { }); } + connectToInstance = async function connectToInstance (app_name) { + const app_info = await this.#ipc_stub({ + method: 'connectToInstance', + parameters: { + app_name, + } + }); + + return AppConnection.from(app_info, { + appInstanceID: this.appInstanceID, + messageTarget: this.messageTarget, + }); + } + parentApp() { return this.#parentAppConnection; } From 2e6315e042eb180450dac02cb706c093db10ca56 Mon Sep 17 00:00:00 2001 From: KernelDeimos Date: Wed, 11 Sep 2024 23:59:25 -0400 Subject: [PATCH 09/18] dev: multi-instance many-to-many app-to-app comms I'll elaborate here because the commit name is confusing. Any time an app gets a connection to another app, either because it launched that app (or was launched by that app) or requested a connection to that app, the ID the app gets to represent the app it's communicating with is now a pseudo app id rather than the app instance ID. This accomplishes two things: 1. It's more secure. There are multiple places where GUI assumes that knowing an app's instance ID means you can do things as that app. 2. Between the same two apps, there may now be more than one connection. This is useful for situations like Phoenix shell talking to the emulator in multiple separate instances to pipe commands. (this is coming later) --- src/gui/src/IPC.js | 16 +++++++++++ src/gui/src/definitions.js | 17 +++++++++--- src/gui/src/helpers/launch_app.js | 2 +- src/gui/src/services/ExecService.js | 29 ++++++++++++++++---- src/gui/src/services/IPCService.js | 42 ++++++++++++++++++++++++++++- src/phoenix/src/main_puter.js | 1 - src/puter-js/src/modules/UI.js | 6 ++++- 7 files changed, 100 insertions(+), 13 deletions(-) diff --git a/src/gui/src/IPC.js b/src/gui/src/IPC.js index beeb0f9058..9a08a50e48 100644 --- a/src/gui/src/IPC.js +++ b/src/gui/src/IPC.js @@ -1439,6 +1439,22 @@ window.addEventListener('message', async (event) => { const { appInstanceID, targetAppInstanceID, targetAppOrigin, contents } = event.data; // TODO: Determine if we should allow the message // TODO: Track message traffic between apps + const svc_ipc = globalThis.services.get('ipc'); + // const svc_exec = globalThis.services() + + const conn = svc_ipc.get_connection(targetAppInstanceID); + if ( conn ) { + conn.send(contents); + // conn.send({ + // msg: 'messageToApp', + // appInstanceID, + // targetAppInstanceID, + // contents, + // }, targetAppOrigin); + return; + } + + console.log(`🔒 App ${appInstanceID} is sending to ${targetAppInstanceID} insecurely`); // pass on the message const target_iframe = window.iframe_for_app_instance(targetAppInstanceID); diff --git a/src/gui/src/definitions.js b/src/gui/src/definitions.js index 5ef2f925c7..22c1496686 100644 --- a/src/gui/src/definitions.js +++ b/src/gui/src/definitions.js @@ -35,6 +35,9 @@ export class Service extends concepts.Service { this.services = a[0].services; return this._init(...a) } + get context () { + return { services: this.services }; + } }; export const PROCESS_INITIALIZING = { i18n_key: 'initializing' }; @@ -139,16 +142,22 @@ export class PortalProcess extends Process { } } - send (channel, object, context) { + send (channel, data, context) { const target = this.references.iframe.contentWindow; - // NEXT: ... + target.postMessage({ + msg: 'messageToApp', + appInstanceID: channel.returnAddress, + targetAppInstanceID: this.uuid, + contents: data, + // }, new URL(this.references.iframe.src).origin); + }, '*'); } - handle_connection (other_process, args) { + handle_connection (connection, args) { const target = this.references.iframe.contentWindow; target.postMessage({ msg: 'connection', - appInstanceID: other_process.uuid, + appInstanceID: connection.uuid, args, }); } diff --git a/src/gui/src/helpers/launch_app.js b/src/gui/src/helpers/launch_app.js index 00e2d2a7dd..bbb47c2a75 100644 --- a/src/gui/src/helpers/launch_app.js +++ b/src/gui/src/helpers/launch_app.js @@ -183,7 +183,7 @@ const launch_app = async (options)=>{ // add parent_app_instance_id to URL if (options.parent_instance_id) { - iframe_url.searchParams.append('puter.parent_instance_id', options.parent_instance_id); + iframe_url.searchParams.append('puter.parent_instance_id', options.parent_pseudo_id); } if(file_signature){ diff --git a/src/gui/src/services/ExecService.js b/src/gui/src/services/ExecService.js index 213705b911..386b6cac57 100644 --- a/src/gui/src/services/ExecService.js +++ b/src/gui/src/services/ExecService.js @@ -20,19 +20,30 @@ export class ExecService extends Service { async launchApp ({ app_name, args }, { ipc_context, msg_id } = {}) { const app = ipc_context?.caller?.app; const process = ipc_context?.caller?.process; - + // This mechanism will be replated with xdrpc soon const child_instance_id = window.uuidv4(); + const svc_ipc = this.services.get('ipc'); + const connection = ipc_context ? svc_ipc.add_connection({ + source: process.uuid, + target: child_instance_id, + }) : undefined; + // The "body" of this method is in a separate file const child_process = await launch_app({ name: app_name, args: args ?? {}, parent_instance_id: app?.appInstanceID, uuid: child_instance_id, + ...(connection ? { + parent_pseudo_id: connection.backward.uuid, + } : {}), }); const send_child_launched_msg = (...a) => { + if ( ! process ) return; + // TODO: (maybe) message process instead of iframe const parent_iframe = process?.references?.iframe; parent_iframe.contentWindow.postMessage({ msg: 'childAppLaunched', @@ -67,9 +78,10 @@ export class ExecService extends Service { window.report_app_closed(child_process.uuid); } }); - + return { - appInstanceID: child_instance_id, + appInstanceID: connection ? + connection.forward.uuid : child_instance_id, usesSDK: true, }; } @@ -96,10 +108,17 @@ export class ExecService extends Service { const options = svc_process.select_by_name(app_name); const process = options[0]; - await process.handle_connection(caller_process, args); + const svc_ipc = this.services.get('ipc'); + const connection = svc_ipc.add_connection({ + source: caller_process.uuid, + target: process.uuid, + }); + + const response = await process.handle_connection( + connection.backward, args); return { - appInstanceID: process.uuid, + appInstanceID: connection.forward.uuid, response, }; } diff --git a/src/gui/src/services/IPCService.js b/src/gui/src/services/IPCService.js index 83ac0bf354..82f7c8f22e 100644 --- a/src/gui/src/services/IPCService.js +++ b/src/gui/src/services/IPCService.js @@ -1,12 +1,52 @@ import { Service } from "../definitions.js"; +class InternalConnection { + constructor ({ source, target, uuid, reverse }, { services }) { + this.services = services; + this.source = source; + this.target = target; + this.uuid = uuid; + this.reverse = reverse; + } + + send (data) { + const svc_process = this.services.get('process'); + const process = svc_process.get_by_uuid(this.target); + const channel = { + returnAddress: this.reverse, + }; + process.send(channel, data); + } +} + export class IPCService extends Service { static description = ` Allows other services to expose methods to apps. ` async _init () { - // + this.connections_ = {}; + } + + add_connection ({ source, target }) { + const uuid = window.uuidv4(); + const r_uuid = window.uuidv4(); + const forward = this.connections_[uuid] = { + source, target, + uuid: uuid, reverse: r_uuid, + }; + const backward = this.connections_[r_uuid] = { + source: target, target: source, + uuid: r_uuid, reverse: uuid, + }; + return { forward, backward }; + } + + get_connection (uuid) { + const entry = this.connections_[uuid]; + if ( ! entry ) return; + if ( entry.object ) return entry.object; + return entry.object = new InternalConnection(entry, this.context); } register_ipc_handler (name, spec) { diff --git a/src/phoenix/src/main_puter.js b/src/phoenix/src/main_puter.js index 01e0339139..d61ef04f88 100644 --- a/src/phoenix/src/main_puter.js +++ b/src/phoenix/src/main_puter.js @@ -53,7 +53,6 @@ window.main_shell = async () => { } }); terminal.on('close', () => { - console.log('Terminal closed; exiting Phoenix...'); puter.exit(); }); diff --git a/src/puter-js/src/modules/UI.js b/src/puter-js/src/modules/UI.js index f8d9a794a3..59a832ccd5 100644 --- a/src/puter-js/src/modules/UI.js +++ b/src/puter-js/src/modules/UI.js @@ -47,6 +47,7 @@ class AppConnection extends EventListener { // Message is from a different AppConnection; ignore it. return; } + // TODO: does this check really make sense? if (event.data.targetAppInstanceID !== this.appInstanceID) { console.error(`AppConnection received message intended for wrong app! appInstanceID=${this.appInstanceID}, target=${event.data.targetAppInstanceID}`); return; @@ -89,7 +90,10 @@ class AppConnection extends EventListener { msg: 'messageToApp', appInstanceID: this.appInstanceID, targetAppInstanceID: this.targetAppInstanceID, - targetAppOrigin: '*', // TODO: Specify this somehow + // Note: there was a TODO comment here about specifying the origin, + // but this should not happen here; the origin should be specified + // on the other side where the expected origin for the app is known. + targetAppOrigin: '*', contents: message, }, this.#puterOrigin); } From a16cfb6f0ccdf291b3c5f4754ad15573f765654f Mon Sep 17 00:00:00 2001 From: KernelDeimos Date: Thu, 12 Sep 2024 13:49:34 -0400 Subject: [PATCH 10/18] tweak: use virtualization-optimized kernel --- src/emulator/image/.gitignore | 1 + src/emulator/image/Dockerfile | 2 +- src/emulator/image/build/.gitignore | 2 -- src/emulator/src/main.js | 4 ++-- 4 files changed, 4 insertions(+), 5 deletions(-) create mode 100644 src/emulator/image/.gitignore delete mode 100644 src/emulator/image/build/.gitignore diff --git a/src/emulator/image/.gitignore b/src/emulator/image/.gitignore new file mode 100644 index 0000000000..378eac25d3 --- /dev/null +++ b/src/emulator/image/.gitignore @@ -0,0 +1 @@ +build diff --git a/src/emulator/image/Dockerfile b/src/emulator/image/Dockerfile index d46fa5974d..e589ab6154 100644 --- a/src/emulator/image/Dockerfile +++ b/src/emulator/image/Dockerfile @@ -2,7 +2,7 @@ FROM i386/alpine:edge RUN apk add --update \ alpine-base bash ncurses shadow curl \ - linux-lts linux-firmware-none linux-headers \ + linux-virt linux-firmware-none linux-headers \ gcc make gcompat musl-dev libx11-dev xinit \ bind-tools \ util-linux \ diff --git a/src/emulator/image/build/.gitignore b/src/emulator/image/build/.gitignore deleted file mode 100644 index d6b7ef32c8..0000000000 --- a/src/emulator/image/build/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore diff --git a/src/emulator/src/main.js b/src/emulator/src/main.js index 1b7d7f9992..b346e86d4b 100644 --- a/src/emulator/src/main.js +++ b/src/emulator/src/main.js @@ -46,10 +46,10 @@ window.onload = async function() }, initrd: { - url: './image/build/boot/initramfs-lts', + url: './image/build/boot/initramfs-virt', }, bzimage: { - url: './image/build/boot/vmlinuz-lts', + url: './image/build/boot/vmlinuz-virt', async: false }, cmdline: 'rw root=/dev/sda init=/sbin/init rootfstype=ext4', From d3e70ebe82ff9d706bcec824aa9898d2f5dc2909 Mon Sep 17 00:00:00 2001 From: KernelDeimos Date: Thu, 12 Sep 2024 17:26:38 -0400 Subject: [PATCH 11/18] dev: add terminal multiplexing This involves establishing the protocol through which phoenix instances run a command on the emulator. The pty is able to communicate with the terminal in both directions. This commit adds logs to be removed later. There are a few things left that this commit does not address: - handling close of delegate process - handling sigint from phoenix to delegate process - closing the connection to twisp --- src/emulator/src/main.js | 92 +++++++++++++++- src/gui/src/definitions.js | 26 ++++- src/gui/src/services/ExecService.js | 7 +- .../providers/EmuCommandProvider.js | 103 ++++++++++++++++-- src/puter-js/src/modules/UI.js | 24 +++- 5 files changed, 235 insertions(+), 17 deletions(-) diff --git a/src/emulator/src/main.js b/src/emulator/src/main.js index b346e86d4b..55ace20fcc 100644 --- a/src/emulator/src/main.js +++ b/src/emulator/src/main.js @@ -9,6 +9,16 @@ const { DataBuilder, } = require("../../puter-wisp/src/exports"); +const status = { + ready: false, +}; + +const state = { + ready_listeners: [], +}; + +let ptyMgr; + class WispClient { constructor ({ packetStream, @@ -23,8 +33,74 @@ class WispClient { } } +const setup_pty = (ptt, pty) => { + console.log('PTY created', pty); + + // resize + ptt.on('ioctl.set', evt => { + console.log('event?', evt); + pty.resize(evt.windowSize); + }); + ptt.TIOCGWINSZ(); + + // data from PTY + pty.on_payload = data => { + ptt.out.write(data); + } + + // data to PTY + (async () => { + // for (;;) { + // const buff = await ptt.in.read(); + // if ( buff === undefined ) continue; + // console.log('this is what ptt in gave', buff); + // pty.send(buff); + // } + const stream = ptt.readableStream; + for await ( const chunk of stream ) { + if ( chunk === undefined ) { + console.error('huh, missing chunk', chunk); + continue; + } + console.log('sending to pty', chunk); + pty.send(chunk); + } + })() +} + + puter.ui.on('connection', event => { + const { conn, accept, reject } = event; + if ( ! status.ready ) { + console.log('status not ready, adding listener'); + state.ready_listeners.push(() => { + console.log('a listener was called'); + conn.postMessage({ + $: 'status', + ...status, + }); + }); + } + accept({ + version: '1.0.0', + status, + }); console.log('emulator got connection event', event); + + const pty_on_first_message = message => { + conn.off('message', pty_on_first_message); + console.log('[!!] message from connection', message); + const pty = ptyMgr.getPTY({ + command: '/bin/bash' + }); + console.log('setting up ptt with...', conn); + const ptt = new XDocumentPTT(conn, { + disableReader: true, + }); + ptt.termios.echo = false; + setup_pty(ptt, pty); + } + conn.on('message', pty_on_first_message); }); window.onload = async function() @@ -83,12 +159,14 @@ window.onload = async function() const virtioStream = NewVirtioFrameStream(byteStream); const wispStream = NewWispPacketStream(virtioStream); + /* const shell = puter.ui.parentApp(); const ptt = new XDocumentPTT(shell, { disableReader: true, }) ptt.termios.echo = false; + */ class PTYManager { static STATE_INIT = { @@ -110,6 +188,13 @@ window.onload = async function() } }, on: function () { + console.log('ready.on called') + status.ready = true; + for ( const listener of state.ready_listeners ) { + console.log('calling listener'); + listener(); + } + return; const pty = this.getPTY(); console.log('PTY created', pty); @@ -176,14 +261,14 @@ window.onload = async function() } } - getPTY () { + getPTY ({ command }) { const streamId = ++this.streamId; const data = new DataBuilder({ leb: true }) .uint8(0x01) .uint32(streamId) .uint8(0x03) .uint16(10) - .utf8('/bin/bash') + .utf8(command) // .utf8('/usr/bin/htop') .build(); const packet = new WispPacket( @@ -235,7 +320,7 @@ window.onload = async function() } } - const ptyMgr = new PTYManager({ + ptyMgr = new PTYManager({ client: new WispClient({ packetStream: wispStream, sendFn: packet => { @@ -249,4 +334,5 @@ window.onload = async function() }) }); ptyMgr.init(); + } diff --git a/src/gui/src/definitions.js b/src/gui/src/definitions.js index 22c1496686..f2e7f8f58a 100644 --- a/src/gui/src/definitions.js +++ b/src/gui/src/definitions.js @@ -18,6 +18,7 @@ */ import { concepts, AdvancedBase } from "@heyputer/putility"; +import TeePromise from "./util/TeePromise.js"; export class Service extends concepts.Service { // TODO: Service todo items @@ -153,13 +154,36 @@ export class PortalProcess extends Process { }, '*'); } - handle_connection (connection, args) { + async handle_connection (connection, args) { const target = this.references.iframe.contentWindow; + const connection_response = new TeePromise(); + window.addEventListener('message', (evt) => { + if ( evt.source !== target ) return; + // Using '$' instead of 'msg' to avoid handling by IPC.js + // (following type-tagged message convention) + if ( evt.data.$ !== 'connection-resp' ) return; + if ( evt.data.connection !== connection.uuid ) return; + if ( evt.data.accept ) { + connection_response.resolve(evt.data.value); + } else { + connection_response.reject(evt.data.value + ?? new Error('Connection rejected')); + } + }); target.postMessage({ msg: 'connection', appInstanceID: connection.uuid, args, }); + const outcome = await Promise.race([ + connection_response, + new Promise((resolve, reject) => { + setTimeout(() => { + reject(new Error('Connection timeout')); + }, 5000); + }) + ]); + return outcome; } }; export class PseudoProcess extends Process { diff --git a/src/gui/src/services/ExecService.js b/src/gui/src/services/ExecService.js index 386b6cac57..5fcad6edaa 100644 --- a/src/gui/src/services/ExecService.js +++ b/src/gui/src/services/ExecService.js @@ -108,6 +108,10 @@ export class ExecService extends Service { const options = svc_process.select_by_name(app_name); const process = options[0]; + if ( ! process ) { + throw new Error(`No process found: ${app_name}`); + } + const svc_ipc = this.services.get('ipc'); const connection = svc_ipc.add_connection({ source: caller_process.uuid, @@ -116,9 +120,10 @@ export class ExecService extends Service { const response = await process.handle_connection( connection.backward, args); - + return { appInstanceID: connection.forward.uuid, + usesSDK: true, response, }; } diff --git a/src/phoenix/src/puter-shell/providers/EmuCommandProvider.js b/src/phoenix/src/puter-shell/providers/EmuCommandProvider.js index 1af4d9d6ea..08a3ef6761 100644 --- a/src/phoenix/src/puter-shell/providers/EmuCommandProvider.js +++ b/src/phoenix/src/puter-shell/providers/EmuCommandProvider.js @@ -1,21 +1,19 @@ +import { TeePromise } from "@heyputer/putility/src/libs/promise"; import { Exit } from "../coreutils/coreutil_lib/exit"; export class EmuCommandProvider { - static AVAILABLE = [ - 'bash', - 'htop', - ]; + static AVAILABLE = { + 'bash': '/bin/bash', + 'htop': '/usr/bin/htop', + }; static EMU_APP_NAME = 'test-emu'; constructor () { this.available = this.constructor.AVAILABLE; - this.emulator = null; } - async aquire_emulator () { - if ( this.emulator ) return this.emulator; - + async aquire_emulator ({ ctx }) { // FUTURE: when we have a way to query instances // without exposing the real instance id /* @@ -27,20 +25,103 @@ export class EmuCommandProvider { */ const conn = await puter.ui.connectToInstance(this.constructor.EMU_APP_NAME); - return this.emulator = conn; + const p_ready = new TeePromise(); + conn.on('message', message => { + if ( message.$ === 'status' ) { + p_ready.resolve(); + } + console.log('[!!] message from the emulator', message); + }); + if ( conn.response.status.ready ) { + p_ready.resolve(); + } + console.log('awaiting emulator ready'); + ctx.externs.out.write('Waiting for emulator...\n'); + await p_ready; + console.log('emulator ready'); + return conn; } async lookup (id, { ctx }) { - if ( ! this.available.includes(id) ) { + if ( ! (id in this.available) ) { return; } - const emu = await this.aquire_emulator(); + const emu = await this.aquire_emulator({ ctx }); if ( ! emu ) { ctx.externs.out.write('No emulator available.\n'); return new Exit(1); } ctx.externs.out.write(`Launching ${id} in emulator ${emu.appInstanceID}\n`); + + return { + name: id, + path: 'Emulator', + execute: this.execute.bind(this, { id, emu, ctx }), + } + } + + async execute ({ id, emu }, ctx) { + // TODO: DRY: most copied from PuterAppCommandProvider + const resize_listener = evt => { + emu.postMessage({ + $: 'ioctl.set', + windowSize: { + rows: evt.detail.rows, + cols: evt.detail.cols, + } + }); + }; + ctx.shell.addEventListener('signal.window-resize', resize_listener); + + // TODO: handle CLOSE -> emu needs to close connection first + const app_close_promise = new TeePromise(); + const sigint_promise = new TeePromise(); + + const decoder = new TextDecoder(); + emu.on('message', message => { + if (message.$ === 'stdout') { + ctx.externs.out.write(decoder.decode(message.data)); + } + if (message.$ === 'chtermios') { + if ( message.termios.echo !== undefined ) { + if ( message.termios.echo ) { + ctx.externs.echo.on(); + } else { + ctx.externs.echo.off(); + } + } + } + }); + + // Repeatedly copy data from stdin to the child, while it's running. + // DRY: Initially copied from PathCommandProvider + let data, done; + const next_data = async () => { + console.log('!~!!!!!'); + ({ value: data, done } = await Promise.race([ + app_close_promise, sigint_promise, ctx.externs.in_.read(), + ])); + console.log('next_data', data, done); + if (data) { + console.log('sending stdin data'); + emu.postMessage({ + $: 'stdin', + data: data, + }); + if (!done) setTimeout(next_data, 0); + } + }; + setTimeout(next_data, 0); + + emu.postMessage({ + $: 'exec', + command: this.available[id], + args: [], + }); + + const never_resolve = new TeePromise(); + await never_resolve; } } diff --git a/src/puter-js/src/modules/UI.js b/src/puter-js/src/modules/UI.js index 59a832ccd5..6175b52da0 100644 --- a/src/puter-js/src/modules/UI.js +++ b/src/puter-js/src/modules/UI.js @@ -25,6 +25,11 @@ class AppConnection extends EventListener { values.appInstanceID, values.usesSDK ); + + // When a connection is established the app is able to + // provide some additional information about itself + connection.response = values.response; + return connection; } @@ -466,12 +471,29 @@ class UI extends EventListener { this.#lastBroadcastValue.set(name, data); } else if ( e.data.msg === 'connection' ) { + e.data.usesSDK = true; // we can safely assume this const conn = AppConnection.from(e.data, { appInstanceID: this.appInstanceID, messageTarget: window.parent, }); + const accept = value => { + this.messageTarget?.postMessage({ + $: 'connection-resp', + connection: e.data.appInstanceID, + accept: true, + value, + }); + }; + const reject = value => { + this.messageTarget?.postMessage({ + $: 'connection-resp', + connection: e.data.appInstanceID, + accept: false, + value, + }); + }; this.emit('connection', { - conn + conn, accept, reject, }); } }); From baeb79b5028679c7a572b58713d7194cbce63512 Mon Sep 17 00:00:00 2001 From: KernelDeimos Date: Thu, 12 Sep 2024 19:04:21 -0400 Subject: [PATCH 12/18] dev: handle pty close Return to phoenix shell when a pty stream is closed by twisp. Pipes in phoenix with commands from the emulator do not appear to be working properly, but something in there is working. --- src/emulator/src/main.js | 11 ++++++++++- .../src/puter-shell/providers/EmuCommandProvider.js | 7 +++++-- src/puter-wisp/src/exports.js | 12 ++++++++++++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/emulator/src/main.js b/src/emulator/src/main.js index 55ace20fcc..67f2daf555 100644 --- a/src/emulator/src/main.js +++ b/src/emulator/src/main.js @@ -91,8 +91,13 @@ puter.ui.on('connection', event => { conn.off('message', pty_on_first_message); console.log('[!!] message from connection', message); const pty = ptyMgr.getPTY({ - command: '/bin/bash' + command: message.command, }); + pty.on_close = () => { + conn.postMessage({ + $: 'pty.close', + }); + } console.log('setting up ptt with...', conn); const ptt = new XDocumentPTT(conn, { disableReader: true, @@ -185,6 +190,10 @@ window.onload = async function() console.log('stream id?', packet.streamId); const pty = this.stream_listeners_[packet.streamId]; pty.on_payload(packet.payload); + }, + [WispPacket.CLOSE.id]: function ({ packet }) { + const pty = this.stream_listeners_[packet.streamId]; + pty.on_close(); } }, on: function () { diff --git a/src/phoenix/src/puter-shell/providers/EmuCommandProvider.js b/src/phoenix/src/puter-shell/providers/EmuCommandProvider.js index 08a3ef6761..4fbdade840 100644 --- a/src/phoenix/src/puter-shell/providers/EmuCommandProvider.js +++ b/src/phoenix/src/puter-shell/providers/EmuCommandProvider.js @@ -5,6 +5,7 @@ export class EmuCommandProvider { static AVAILABLE = { 'bash': '/bin/bash', 'htop': '/usr/bin/htop', + 'emu-sort': '/usr/bin/sort', }; static EMU_APP_NAME = 'test-emu'; @@ -93,6 +94,9 @@ export class EmuCommandProvider { } } } + if (message.$ === 'pty.close') { + app_close_promise.resolve(); + } }); // Repeatedly copy data from stdin to the child, while it's running. @@ -121,7 +125,6 @@ export class EmuCommandProvider { args: [], }); - const never_resolve = new TeePromise(); - await never_resolve; + await app_close_promise; } } diff --git a/src/puter-wisp/src/exports.js b/src/puter-wisp/src/exports.js index 88b5028324..ae39724e1c 100644 --- a/src/puter-wisp/src/exports.js +++ b/src/puter-wisp/src/exports.js @@ -223,6 +223,18 @@ const wisp_types = [ } } }, + { + id: 4, + label: 'CLOSE', + describe: ({ attributes }) => { + return `reason: ${attributes.code}`; + }, + getAttributes ({ payload }) { + return { + code: payload[0], + } + } + }, { // TODO: extension types should not be hardcoded here id: 0xf0, From 191977f9e3575d2dfda960c1dcd43ff76e35d878 Mon Sep 17 00:00:00 2001 From: KernelDeimos Date: Mon, 16 Sep 2024 16:22:25 -0400 Subject: [PATCH 13/18] dev: add puter-linux app to db migrations --- .../database/SqliteDatabaseAccessService.js | 35 +++++++++++-------- .../sqlite_setup/0027_emulator-app.dbmig.js | 23 ++++++++++++ src/gui/src/services/ExecService.js | 2 +- src/gui/src/services/ProcessService.js | 2 +- .../providers/EmuCommandProvider.js | 2 +- 5 files changed, 46 insertions(+), 18 deletions(-) create mode 100644 src/backend/src/services/database/sqlite_setup/0027_emulator-app.dbmig.js diff --git a/src/backend/src/services/database/SqliteDatabaseAccessService.js b/src/backend/src/services/database/SqliteDatabaseAccessService.js index 5458cbf370..2c6470b4bc 100644 --- a/src/backend/src/services/database/SqliteDatabaseAccessService.js +++ b/src/backend/src/services/database/SqliteDatabaseAccessService.js @@ -44,21 +44,6 @@ class SqliteDatabaseAccessService extends BaseDatabaseAccessService { this.db = new Database(this.config.path); - // Database upgrade logic - const HIGHEST_VERSION = 24; - const TARGET_VERSION = (() => { - const args = Context.get('args'); - if ( args['database-target-version'] ) { - return parseInt(args['database-target-version']); - } - return HIGHEST_VERSION; - })(); - - const [{ user_version }] = do_setup - ? [{ user_version: -1 }] - : await this._read('PRAGMA user_version'); - this.log.info('database version: ' + user_version); - const upgrade_files = []; const available_migrations = [ @@ -138,8 +123,28 @@ class SqliteDatabaseAccessService extends BaseDatabaseAccessService { [23, [ '0026_user-groups.dbmig.js', ]], + [24, [ + '0027_emulator-app.dbmig.js', + ]], ]; + // Database upgrade logic + const HIGHEST_VERSION = + available_migrations[available_migrations.length - 1][0] + 1; + const TARGET_VERSION = (() => { + const args = Context.get('args'); + if ( args['database-target-version'] ) { + return parseInt(args['database-target-version']); + } + return HIGHEST_VERSION; + })(); + + const [{ user_version }] = do_setup + ? [{ user_version: -1 }] + : await this._read('PRAGMA user_version'); + this.log.info('database version: ' + user_version); + + for ( const [v_lt_or_eq, files] of available_migrations ) { if ( v_lt_or_eq + 1 >= TARGET_VERSION && TARGET_VERSION !== HIGHEST_VERSION ) { this.log.noticeme(`Early exit: target version set to ${TARGET_VERSION}`); diff --git a/src/backend/src/services/database/sqlite_setup/0027_emulator-app.dbmig.js b/src/backend/src/services/database/sqlite_setup/0027_emulator-app.dbmig.js new file mode 100644 index 0000000000..0e3b5ad13f --- /dev/null +++ b/src/backend/src/services/database/sqlite_setup/0027_emulator-app.dbmig.js @@ -0,0 +1,23 @@ +const insert = async (tbl, subject) => { + const keys = Object.keys(subject); + + await write( + 'INSERT INTO `'+ tbl +'` ' + + '(' + keys.map(key => key).join(', ') + ') ' + + 'VALUES (' + keys.map(() => '?').join(', ') + ')', + keys.map(key => subject[key]) + ); +} + +await insert('apps', { + uid: 'app-fbbdb72b-ad08-4cb4-86a1-de0f27cf2e1e', + owner_user_id: 1, + name: 'puter-linux', + index_url: 'https://builtins.namespaces.puter.com/emulator', + title: 'Puter Linux', + description: 'Linux emulator for Puter', + approved_for_listing: 1, + approved_for_opening_items: 1, + approved_for_incentive_program: 0, + timestamp: '2020-01-01 00:00:00', +}); diff --git a/src/gui/src/services/ExecService.js b/src/gui/src/services/ExecService.js index 5fcad6edaa..ed6813a791 100644 --- a/src/gui/src/services/ExecService.js +++ b/src/gui/src/services/ExecService.js @@ -100,7 +100,7 @@ export class ExecService extends Service { if ( caller_process.name !== 'phoenix' ) { throw new Error('Connection not allowed.'); } - if ( app_name !== 'test-emu' ) { + if ( app_name !== 'puter-linux' ) { throw new Error('Connection not allowed.'); } diff --git a/src/gui/src/services/ProcessService.js b/src/gui/src/services/ProcessService.js index d41404ee3c..a6ac38a69d 100644 --- a/src/gui/src/services/ProcessService.js +++ b/src/gui/src/services/ProcessService.js @@ -23,7 +23,7 @@ const NULL_UUID = '00000000-0000-0000-0000-000000000000'; export class ProcessService extends Service { static INITRC = [ - 'test-emu' + 'puter-linux' ]; async _init () { diff --git a/src/phoenix/src/puter-shell/providers/EmuCommandProvider.js b/src/phoenix/src/puter-shell/providers/EmuCommandProvider.js index 4fbdade840..81f90cea31 100644 --- a/src/phoenix/src/puter-shell/providers/EmuCommandProvider.js +++ b/src/phoenix/src/puter-shell/providers/EmuCommandProvider.js @@ -8,7 +8,7 @@ export class EmuCommandProvider { 'emu-sort': '/usr/bin/sort', }; - static EMU_APP_NAME = 'test-emu'; + static EMU_APP_NAME = 'puter-linux'; constructor () { this.available = this.constructor.AVAILABLE; From 86f2dc485fa28d99192b843acc51804d2e1a2a28 Mon Sep 17 00:00:00 2001 From: KernelDeimos Date: Mon, 16 Sep 2024 17:45:14 -0400 Subject: [PATCH 14/18] dev: prep for network build --- src/emulator/src/main.js | 23 ++++++++++++++++++- .../providers/EmuCommandProvider.js | 13 +++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/emulator/src/main.js b/src/emulator/src/main.js index 67f2daf555..8a04929947 100644 --- a/src/emulator/src/main.js +++ b/src/emulator/src/main.js @@ -110,6 +110,23 @@ puter.ui.on('connection', event => { window.onload = async function() { + let emu_config; try { + emu_config = await puter.fs.read('config.json'); + } catch (e) {} + + if ( ! emu_config ) { + await puter.fs.write('config.json', JSON.stringify({})); + emu_config = {}; + } + + if ( emu_config instanceof Blob ) { + emu_config = await emu_config.text(); + } + + if ( typeof emu_config === 'string' ) { + emu_config = JSON.parse(emu_config); + } + const resp = await fetch( './image/build/rootfs.bin' ); @@ -152,10 +169,14 @@ window.onload = async function() // bzimage_initrd_from_filesystem: true, autostart: true, - network_relay_url: "wisp://127.0.0.1:3000", + network_relay_url: emu_config.network_relay ?? "wisp://127.0.0.1:3000", virtio_console: true, }); + emulator.add_listener('download-error', function(e) { + status.missing_files || (status.missing_files = []); + status.missing_files.push(e.file_name); + }); const decoder = new TextDecoder(); const byteStream = NewCallbackByteStream(); diff --git a/src/phoenix/src/puter-shell/providers/EmuCommandProvider.js b/src/phoenix/src/puter-shell/providers/EmuCommandProvider.js index 81f90cea31..3c2257c2a6 100644 --- a/src/phoenix/src/puter-shell/providers/EmuCommandProvider.js +++ b/src/phoenix/src/puter-shell/providers/EmuCommandProvider.js @@ -36,6 +36,19 @@ export class EmuCommandProvider { if ( conn.response.status.ready ) { p_ready.resolve(); } + console.log('status from emu', conn.response); + if ( conn.response.status.missing_files ) { + const pfx = '\x1B[31;1m┃\x1B[0m '; + ctx.externs.out.write('\n'); + ctx.externs.out.write('\x1B[31;1m┃ Emulator is missing files:\x1B[0m\n'); + for (const file of conn.response.status.missing_files) { + ctx.externs.out.write(pfx+`- ${file}\n`); + } + ctx.externs.out.write(pfx+'\n'); + ctx.externs.out.write(pfx+'\x1B[33;1mDid you run `./tools/build_v86.sh`?\x1B[0m\n'); + ctx.externs.out.write('\n'); + return; + } console.log('awaiting emulator ready'); ctx.externs.out.write('Waiting for emulator...\n'); await p_ready; From 9c897fdf59e0591a6c9b1adc58ee1f4cf9db6d91 Mon Sep 17 00:00:00 2001 From: KernelDeimos Date: Tue, 17 Sep 2024 17:15:50 -0400 Subject: [PATCH 15/18] git: add epoxy-tls submodule --- .gitmodules | 3 +++ submodules/epoxy-tls | 1 + 2 files changed, 4 insertions(+) create mode 160000 submodules/epoxy-tls diff --git a/.gitmodules b/.gitmodules index c2dfcc1d99..0000f1d57a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "submodules/twisp"] path = submodules/twisp url = git@github.com:MercuryWorkshop/twisp.git +[submodule "submodules/epoxy-tls"] + path = submodules/epoxy-tls + url = git@github.com:MercuryWorkshop/epoxy-tls.git diff --git a/submodules/epoxy-tls b/submodules/epoxy-tls new file mode 160000 index 0000000000..7fdacb2623 --- /dev/null +++ b/submodules/epoxy-tls @@ -0,0 +1 @@ +Subproject commit 7fdacb26237a0a69faacd1c08d746584fbd98f94 From 9816411451aa52aa20181c0bbe909307fc0eef57 Mon Sep 17 00:00:00 2001 From: KernelDeimos Date: Wed, 18 Sep 2024 14:48:55 -0400 Subject: [PATCH 16/18] dev: add build script for relay --- tools/build_relay.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 tools/build_relay.sh diff --git a/tools/build_relay.sh b/tools/build_relay.sh new file mode 100644 index 0000000000..1990efcc46 --- /dev/null +++ b/tools/build_relay.sh @@ -0,0 +1,17 @@ + +#!/bin/bash + +start_dir=$(pwd) +cleanup() { + cd "$start_dir" +} +trap cleanup ERR EXIT +set -e + +echo -e "\x1B[36;1m<<< Building epoxy-tls >>>\x1B[0m" + +cd submodules/epoxy-tls +rustup install nightly +rustup override set nightly +cargo b -r +cd - From dfb2523bd6c77599018f7cc009a72ff42e675f88 Mon Sep 17 00:00:00 2001 From: KernelDeimos Date: Wed, 18 Sep 2024 16:19:48 -0400 Subject: [PATCH 17/18] fix: upsert subdomain check to insert only --- src/backend/src/om/entitystorage/SubdomainES.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/backend/src/om/entitystorage/SubdomainES.js b/src/backend/src/om/entitystorage/SubdomainES.js index 2932db5ae9..27a543e0ca 100644 --- a/src/backend/src/om/entitystorage/SubdomainES.js +++ b/src/backend/src/om/entitystorage/SubdomainES.js @@ -39,7 +39,9 @@ class SubdomainES extends BaseES { } }, async upsert (entity, extra) { - await this._check_max_subdomains(); + if ( ! extra.old_entity ) { + await this._check_max_subdomains(); + } return await this.upstream.upsert(entity, extra); }, From 6ec3c8b6536b07d5a342aeb8631c7f8b32ff04ef Mon Sep 17 00:00:00 2001 From: KernelDeimos Date: Wed, 18 Sep 2024 16:20:11 -0400 Subject: [PATCH 18/18] dev: add emulator prod deployment --- src/emulator/src/main.js | 10 +++++++--- src/emulator/webpack.config.js | 4 ++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/emulator/src/main.js b/src/emulator/src/main.js index 8a04929947..6966b9e2fa 100644 --- a/src/emulator/src/main.js +++ b/src/emulator/src/main.js @@ -1,5 +1,9 @@ "use strict"; +console.log(`emulator running in mode: ${MODE}`) + +const PATH_V86 = MODE === 'dev' ? '/vendor/v86' : './vendor/v86'; + const { XDocumentPTT } = require("../../phoenix/src/pty/XDocumentPTT"); const { NewWispPacketStream, @@ -132,15 +136,15 @@ window.onload = async function() ); const arrayBuffer = await resp.arrayBuffer(); var emulator = window.emulator = new V86({ - wasm_path: "/vendor/v86/v86.wasm", + wasm_path: PATH_V86 + "/v86.wasm", memory_size: 512 * 1024 * 1024, vga_memory_size: 2 * 1024 * 1024, screen_container: document.getElementById("screen_container"), bios: { - url: "/vendor/v86/bios/seabios.bin", + url: PATH_V86 + "/bios/seabios.bin", }, vga_bios: { - url: "/vendor/v86/bios/vgabios.bin", + url: PATH_V86 + "/bios/vgabios.bin", }, initrd: { diff --git a/src/emulator/webpack.config.js b/src/emulator/webpack.config.js index 6983e59a3a..6dee896116 100644 --- a/src/emulator/webpack.config.js +++ b/src/emulator/webpack.config.js @@ -1,4 +1,5 @@ const HtmlWebpackPlugin = require('html-webpack-plugin'); +const DefinePlugin = require('webpack').DefinePlugin; module.exports = { entry: [ @@ -8,5 +9,8 @@ module.exports = { new HtmlWebpackPlugin({ template: 'assets/template.html' }), + new DefinePlugin({ + MODE: JSON.stringify(process.env.MODE ?? 'dev') + }), ] };