diff options
| author | Andre Bezerra <amelco.herman@gmail.com> | 2026-10-28 16:30:44 -0300 |
|---|---|---|
| committer | Andre Bezerra <amelco.herman@gmail.com> | 2026-10-28 16:30:44 -0300 |
| commit | 3db6dfa630a3699953babe0b02fbe60c20b81cb4 (patch) | |
| tree | 251a35322d55a1b35f363982c2ca8d4117c1a300 | |
| parent | 5f69ee978ed63fc8fc6e5cb48e2c7b0a71a865bd (diff) | |
add script to bundle everything in a single executable
| -rw-r--r-- | .gitignore | 2 | ||||
| -rw-r--r-- | README.md | 41 | ||||
| -rwxr-xr-x | package.sh | 131 | ||||
| -rw-r--r-- | packaging/launcher.c | 612 | ||||
| -rw-r--r-- | packaging/licenses/DEMUCS-LICENSE.txt | 21 | ||||
| -rw-r--r-- | packaging/licenses/HTDEMUCS-6S-ONNX-LICENSE.txt | 21 | ||||
| -rwxr-xr-x | packaging/mkbundle.py | 270 |
7 files changed, 1098 insertions, 0 deletions
@@ -1,4 +1,6 @@ /stemsep +/build/ +/dist/ /models/*.onnx /*_stems/ tmp/ @@ -75,6 +75,47 @@ successfully; normal errors and interruptions remove incomplete files. An existing output directory is never overwritten. A process crash or power loss can leave an incomplete directory that must be removed manually. +## Single-file distribution (Linux x86-64) + +`./package.sh --portable` produces `dist/stemsep-linux-x86_64`, one executable +containing `stemsep`, the ONNX Runtime shared library, the 258 MB model, and +the license notices. The payload is a zstd-compressed archive appended to a +small statically linked launcher. `mpv` is not part of the bundle: playback +(`-p`) still requires `mpv` in `PATH`. + +```sh +./package.sh --portable +./dist/stemsep-linux-x86_64 -wav /path/to/song.wav -m song +``` + +The first run unpacks the payload into a per-user cache; later runs reuse it. +The cache is `$STEMSEP_CACHE` if set, otherwise `$XDG_CACHE_HOME/stemsep`, or +`~/.cache/stemsep`. Extraction takes about a second and uses roughly 270 MB. +Concurrent first runs are serialized with a lock file. Remove the cache with +`rm -rf ~/.cache/stemsep`. + +The bundle accepts three informational options before the normal arguments: + +- `--stemsep-extract`: print the extraction directory and exit. +- `--stemsep-licenses`: print the bundled license directory and exit. +- `--stemsep-version`: print the bundle build id and exit. + +Requirements and notes: + +- Linux x86-64 with glibc 2.28 or newer. `--portable` builds `stemsep` in a + `gcc:7` container for that baseline. ONNX Runtime itself requires glibc + 2.27, so older glibc systems and musl-based distributions such as Alpine + are not supported. +- No root privileges, FUSE, or runtime installation step are needed. +- `ZSTD_LEVEL` (or `--zstd-level`) controls compression; the default is 19 and + produces a ~140 MB file. Level 0 disables compression and yields a ~280 MB + file. +- `STEMSEP_MODEL` still overrides the bundled model. +- `./package.sh --no-build` packages an existing `./stemsep`, and + `--skip-model-check` permits packaging a model other than the pinned one. +- Packaging needs `gcc` with a static libc and `libzstd.a`, `python3`, `zstd`, + and Docker for `--portable`. + ## WAV support Input must be an uncompressed RIFF/WAVE file with one or two channels. The diff --git a/package.sh b/package.sh new file mode 100755 index 0000000..28d4d8a --- /dev/null +++ b/package.sh @@ -0,0 +1,131 @@ +#!/bin/sh +# Build a single-file Linux distribution of stemsep. +# +# ./package.sh native build (runs on systems with a recent glibc) +# ./package.sh --portable build stemsep in the gcc:7 (glibc 2.28) container +# ./package.sh --no-build package the existing ./stemsep +# +# The output embeds stemsep, libonnxruntime, the ONNX model, and license +# notices. It extracts itself into a cache directory on first run. +set -eu +cd -- "$(dirname -- "$0")" + +case "$(uname -s):$(uname -m)" in + Linux:x86_64) ;; + *) + echo "error: only Linux x86-64 bundles are supported" >&2 + exit 1 + ;; +esac + +output=dist/stemsep-linux-x86_64 +zstd_level=${ZSTD_LEVEL:-19} +portable=0 +skip_build=0 +check_model=1 +image=${DOCKER_IMAGE:-gcc:7} +model=models/htdemucs_6s.onnx +model_sha256=48f8e84945579f8ab340e083339e9221e03785dbe733a52c388200b6d3ca779a + +usage() +{ + cat <<'EOF' +Usage: ./package.sh [OPTIONS] + + --portable build stemsep in a Docker container based on glibc 2.28 + --no-build do not rebuild stemsep; package the existing binary + --output PATH output file (default: dist/stemsep-linux-x86_64) + --zstd-level N compression level, 0 disables compression (default: 19) + --skip-model-check do not verify the pinned model SHA-256 + -h, --help show this help +EOF +} + +while [ $# -gt 0 ]; do + case "$1" in + --portable) portable=1 ;; + --no-build) skip_build=1 ;; + --output) + [ $# -ge 2 ] || { echo "error: --output requires a value" >&2; exit 2; } + output=$2 + shift + ;; + --zstd-level) + [ $# -ge 2 ] || { echo "error: --zstd-level requires a value" >&2; exit 2; } + zstd_level=$2 + shift + ;; + --skip-model-check) check_model=0 ;; + -h|--help) usage; exit 0 ;; + *) echo "error: unknown option '$1'" >&2; usage >&2; exit 2 ;; + esac + shift +done + +if [ "$skip_build" -eq 0 ]; then + if [ "$portable" -eq 1 ]; then + command -v docker >/dev/null 2>&1 || { + echo "error: --portable requires docker" >&2 + exit 1 + } + echo "Building stemsep in $image (x86-64, glibc 2.28 baseline)..." + docker run --rm --platform linux/amd64 \ + -u "$(id -u):$(id -g)" \ + -v "$PWD:/work" -w /work "$image" ./build.sh + else + ./build.sh + fi +fi + +[ -x ./stemsep ] || { + echo "error: ./stemsep is missing; run ./build.sh first" >&2 + exit 1 +} +[ -f "$model" ] || { + echo "error: $model is missing; see README.md" >&2 + exit 1 +} + +if [ "$check_model" -eq 1 ]; then + actual=$(sha256sum "$model" | cut -d ' ' -f 1) + if [ "$actual" != "$model_sha256" ]; then + echo "error: $model has SHA-256 $actual" >&2 + echo " expected $model_sha256 (use --skip-model-check to override)" >&2 + exit 1 + fi +fi + +command -v python3 >/dev/null 2>&1 || { + echo "error: python3 is required to package the bundle" >&2 + exit 1 +} + +mkdir -p build dist +cc=${CC:-gcc} +echo "Building static launcher..." +$cc -std=c11 -O2 -DNDEBUG -Wall -Wextra -Wpedantic -static \ + -o build/launcher packaging/launcher.c -lzstd -pthread + +echo "Packing bundle (zstd level $zstd_level)..." +set -- \ + --launcher build/launcher \ + --output "$output" \ + --zstd-level "$zstd_level" \ + --add stemsep=./stemsep \ + --add third_party/onnxruntime/lib/libonnxruntime.so.1.21.0=third_party/onnxruntime/lib/libonnxruntime.so.1.21.0 \ + --symlink third_party/onnxruntime/lib/libonnxruntime.so.1=libonnxruntime.so.1.21.0 \ + --add "$model=$model" \ + --add licenses/ONNX-Runtime-LICENSE.txt=third_party/onnxruntime/LICENSE \ + --add licenses/ONNX-Runtime-ThirdPartyNotices.txt=third_party/onnxruntime/ThirdPartyNotices.txt \ + --add licenses/htdemucs-6s-model-README.md=models/README.md \ + --add licenses/Demucs-LICENSE.txt=packaging/licenses/DEMUCS-LICENSE.txt \ + --add licenses/htdemucs-6s-ONNX-LICENSE.txt=packaging/licenses/HTDEMUCS-6S-ONNX-LICENSE.txt \ + --add README.md=README.md +if [ "$check_model" -eq 1 ]; then + set -- "$@" --checksum "$model=$model_sha256" +fi +python3 packaging/mkbundle.py "$@" + +echo "Bundle: $output" +echo "Run './$output' on Linux x86-64. First run extracts to \$STEMSEP_CACHE," +echo "or \$XDG_CACHE_HOME/stemsep, or ~/.cache/stemsep." diff --git a/packaging/launcher.c b/packaging/launcher.c new file mode 100644 index 0000000..70ac9ea --- /dev/null +++ b/packaging/launcher.c @@ -0,0 +1,612 @@ +/* + * Self-contained launcher for the stemsep single-file Linux bundle. + * + * The bundle is an ELF launcher followed by a zstd-compressed archive and a + * 64-byte trailer. On first run the archive is unpacked into a cache + * directory; later runs reuse it. The extracted `stemsep` keeps the layout it + * expects at runtime: + * + * <cache>/bundle-<id>/stemsep + * <cache>/bundle-<id>/third_party/onnxruntime/lib/libonnxruntime.so.1 + * <cache>/bundle-<id>/models/htdemucs_6s.onnx + * + * Only Linux/x86-64 is supported by the bundled ONNX Runtime build. + */ +#define _GNU_SOURCE + +#include <errno.h> +#include <fcntl.h> +#include <stdarg.h> +#include <stdbool.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/file.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +#include <zstd.h> + +#define TRAILER_SIZE 64U +#define TRAILER_MAGIC "STMSF001" +#define TRAILER_FLAGS 0U +#define ARCHIVE_MAGIC "STEMSEPA" +#define ARCHIVE_VERSION 1U +#define PATH_BUFFER 4096U + +enum entry_type { + ENTRY_FILE = 0, + ENTRY_SYMLINK = 1, + ENTRY_DIRECTORY = 2 +}; + +enum entry_compression { + COMP_NONE = 0, + COMP_ZSTD = 1 +}; + +struct trailer { + uint64_t archive_offset; + uint64_t archive_size; + char build_id[33]; +}; + +static void fail(const char *format, ...) +{ + va_list arguments; + + fputs("stemsep bundle: ", stderr); + va_start(arguments, format); + vfprintf(stderr, format, arguments); + va_end(arguments); + fputc('\n', stderr); + exit(127); +} + +static void *xmalloc(size_t size) +{ + void *pointer = malloc(size); + + if (pointer == NULL) { + fail("out of memory"); + } + return pointer; +} + +static char *xstrdup(const char *text) +{ + char *copy = strdup(text); + + if (copy == NULL) { + fail("out of memory"); + } + return copy; +} + +static char *join_path(const char *left, const char *right) +{ + size_t left_length = strlen(left); + size_t right_length = strlen(right); + bool separator = left_length > 0U && left[left_length - 1U] != '/'; + char *result = xmalloc(left_length + (separator ? 1U : 0U) + + right_length + 1U); + + memcpy(result, left, left_length); + if (separator) { + result[left_length++] = '/'; + } + memcpy(result + left_length, right, right_length + 1U); + return result; +} + +static uint16_t load_u16(const unsigned char *data) +{ + return (uint16_t)((uint16_t)data[0] | ((uint16_t)data[1] << 8)); +} + +static uint32_t load_u32(const unsigned char *data) +{ + return (uint32_t)data[0] | ((uint32_t)data[1] << 8) | + ((uint32_t)data[2] << 16) | ((uint32_t)data[3] << 24); +} + +static uint64_t load_u64(const unsigned char *data) +{ + return (uint64_t)load_u32(data) | ((uint64_t)load_u32(data + 4) << 32); +} + +static void write_all(int fd, const void *data, size_t length, const char *path) +{ + const unsigned char *cursor = data; + + while (length > 0U) { + ssize_t written = write(fd, cursor, length); + + if (written < 0) { + if (errno == EINTR) { + continue; + } + fail("cannot write '%s': %s", path, strerror(errno)); + } + cursor += (size_t)written; + length -= (size_t)written; + } +} + +static void make_directories(const char *path, mode_t mode) +{ + size_t length = strlen(path); + char *copy; + + if (length == 0U || strcmp(path, "/") == 0) { + return; + } + copy = xstrdup(path); + if (copy[length - 1U] == '/') { + copy[length - 1U] = '\0'; + length--; + } + for (char *cursor = copy + 1; ; cursor++) { + if (*cursor == '/' || *cursor == '\0') { + char saved = *cursor; + + *cursor = '\0'; + if (mkdir(copy, mode) != 0 && errno != EEXIST) { + fail("cannot create directory '%s': %s", copy, + strerror(errno)); + } + *cursor = saved; + if (saved == '\0') { + break; + } + } + } + free(copy); +} + +static void make_parent_directories(const char *path) +{ + char *copy = xstrdup(path); + char *slash = strrchr(copy, '/'); + + if (slash != NULL && slash != copy) { + *slash = '\0'; + make_directories(copy, 0700); + } + free(copy); +} + +static bool path_is_safe(const char *path) +{ + const char *cursor = path; + + if (path[0] == '\0' || path[0] == '/') { + return false; + } + while (*cursor != '\0') { + const char *end = strchr(cursor, '/'); + size_t length = end != NULL ? (size_t)(end - cursor) : strlen(cursor); + + if (length == 0U || + (length == 1U && cursor[0] == '.') || + (length == 2U && cursor[0] == '.' && cursor[1] == '.')) { + return false; + } + if (end == NULL) { + break; + } + cursor = end + 1; + } + return true; +} + +static char *self_path(void) +{ + size_t capacity = 256U; + + for (;;) { + char *buffer = xmalloc(capacity); + ssize_t length = readlink("/proc/self/exe", buffer, capacity - 1U); + + if (length < 0) { + fail("cannot resolve /proc/self/exe: %s", strerror(errno)); + } + if ((size_t)length < capacity - 1U) { + buffer[length] = '\0'; + return buffer; + } + free(buffer); + if (capacity > (1U << 20)) { + fail("executable path is too long"); + } + capacity *= 2U; + } +} + +static void read_exact_at(int fd, void *buffer, size_t length, off_t offset, + const char *description) +{ + unsigned char *cursor = buffer; + + while (length > 0U) { + ssize_t received = pread(fd, cursor, length, offset); + + if (received < 0) { + if (errno == EINTR) { + continue; + } + fail("cannot read %s: %s", description, strerror(errno)); + } + if (received == 0) { + fail("unexpected end of file while reading %s", description); + } + cursor += (size_t)received; + length -= (size_t)received; + offset += received; + } +} + +static struct trailer read_trailer(int fd, off_t file_size) +{ + unsigned char raw[TRAILER_SIZE]; + struct trailer result; + + if (file_size < (off_t)TRAILER_SIZE) { + fail("file is too small to be a stemsep bundle"); + } + read_exact_at(fd, raw, sizeof raw, file_size - (off_t)TRAILER_SIZE, + "bundle trailer"); + if (memcmp(raw, TRAILER_MAGIC, 8U) != 0) { + fail("this file is not a stemsep bundle (bad trailer magic)"); + } + result.archive_offset = load_u64(raw + 8); + result.archive_size = load_u64(raw + 16); + if (load_u64(raw + 24) != TRAILER_FLAGS) { + fail("unsupported bundle flags"); + } + memcpy(result.build_id, raw + 32, 32U); + result.build_id[32] = '\0'; + + if (result.archive_offset % 4096U != 0U || + result.archive_offset > (uint64_t)file_size || + result.archive_size > (uint64_t)file_size - result.archive_offset || + result.archive_size == 0U || + result.archive_offset + result.archive_size > + (uint64_t)file_size - TRAILER_SIZE) { + fail("bundle layout is corrupt"); + } + for (size_t index = 0U; index < 32U; index++) { + char character = result.build_id[index]; + + if (!((character >= '0' && character <= '9') || + (character >= 'a' && character <= 'f'))) { + fail("bundle build id is invalid"); + } + } + return result; +} + +static char *cache_root(void) +{ + const char *value = getenv("STEMSEP_CACHE"); + char fallback[64]; + + if (value != NULL && value[0] != '\0') { + return xstrdup(value); + } + value = getenv("XDG_CACHE_HOME"); + if (value != NULL && value[0] != '\0') { + return join_path(value, "stemsep"); + } + value = getenv("HOME"); + if (value != NULL && value[0] != '\0') { + return join_path(value, ".cache/stemsep"); + } + snprintf(fallback, sizeof fallback, "/tmp/stemsep-%lu", + (unsigned long)getuid()); + return xstrdup(fallback); +} + +static bool marker_matches(const char *directory, const char *build_id) +{ + char *marker = join_path(directory, ".complete"); + char contents[33]; + int fd = open(marker, O_RDONLY | O_CLOEXEC); + bool matches = false; + + free(marker); + if (fd < 0) { + return false; + } + ssize_t length = read(fd, contents, 32U); + close(fd); + if (length == 32 && memcmp(contents, build_id, 32U) == 0) { + matches = true; + } + return matches; +} + +struct archive_reader { + const unsigned char *data; + size_t size; + size_t position; +}; + +static void require_bytes(const struct archive_reader *reader, size_t count) +{ + if (count > reader->size - reader->position) { + fail("bundle archive is truncated"); + } +} + +static void decompress_zstd(int fd, const unsigned char *source, + size_t source_size, uint64_t expected_size, + const char *path) +{ + ZSTD_DStream *stream = ZSTD_createDStream(); + ZSTD_inBuffer input; + unsigned char output[262144]; + uint64_t total = 0U; + + if (stream == NULL) { + fail("cannot allocate zstd decompressor"); + } + if (ZSTD_isError(ZSTD_initDStream(stream))) { + fail("cannot initialize zstd decompressor"); + } + input.src = source; + input.size = source_size; + input.pos = 0U; + + for (;;) { + ZSTD_outBuffer produced; + size_t status; + + produced.dst = output; + produced.size = sizeof output; + produced.pos = 0U; + status = ZSTD_decompressStream(stream, &produced, &input); + if (ZSTD_isError(status)) { + fail("cannot decompress '%s': %s", path, + ZSTD_getErrorName(status)); + } + if (produced.pos > 0U) { + write_all(fd, output, produced.pos, path); + total += produced.pos; + } + if (status == 0U) { + if (input.pos != input.size) { + fail("compressed data for '%s' has trailing bytes", path); + } + break; + } + if (input.pos == input.size && produced.pos == 0U) { + fail("compressed data for '%s' is truncated", path); + } + } + ZSTD_freeDStream(stream); + if (total != expected_size) { + fail("decompressed size mismatch for '%s'", path); + } +} + +static void extract_archive(int bundle_fd, const struct trailer *trailer, + const char *destination) +{ + const unsigned char *archive; + struct archive_reader reader; + uint32_t entry_count; + + archive = mmap(NULL, (size_t)trailer->archive_size, PROT_READ, + MAP_PRIVATE, bundle_fd, (off_t)trailer->archive_offset); + if (archive == MAP_FAILED) { + fail("cannot map bundle archive: %s", strerror(errno)); + } + if (trailer->archive_size < 16U || + memcmp(archive, ARCHIVE_MAGIC, 8U) != 0) { + fail("bundle archive has a bad header"); + } + if (load_u32(archive + 8) != ARCHIVE_VERSION) { + fail("unsupported bundle archive version"); + } + entry_count = load_u32(archive + 12); + + reader.data = archive; + reader.size = (size_t)trailer->archive_size; + reader.position = 16U; + + for (uint32_t index = 0U; index < entry_count; index++) { + uint16_t path_length; + uint8_t type; + uint8_t compression; + uint16_t mode; + uint64_t original_size; + uint64_t stored_size; + char path[PATH_BUFFER]; + char *target; + const unsigned char *payload; + + require_bytes(&reader, 22U); + path_length = load_u16(reader.data + reader.position); + type = reader.data[reader.position + 2U]; + compression = reader.data[reader.position + 3U]; + mode = load_u16(reader.data + reader.position + 4U); + original_size = load_u64(reader.data + reader.position + 6U); + stored_size = load_u64(reader.data + reader.position + 14U); + reader.position += 22U; + + if (path_length == 0U || path_length >= PATH_BUFFER) { + fail("bundle archive contains an invalid path length"); + } + require_bytes(&reader, path_length); + memcpy(path, reader.data + reader.position, path_length); + path[path_length] = '\0'; + reader.position += path_length; + if (!path_is_safe(path)) { + fail("bundle archive contains an unsafe path '%s'", path); + } + target = join_path(destination, path); + + if (type == ENTRY_DIRECTORY) { + if (stored_size != 0U || original_size != 0U) { + fail("directory entry '%s' has data", path); + } + make_directories(target, 0755); + } else if (type == ENTRY_SYMLINK) { + if (original_size == 0U || original_size >= PATH_BUFFER || + compression != COMP_NONE || stored_size != original_size) { + fail("symlink entry '%s' is invalid", path); + } + require_bytes(&reader, (size_t)stored_size); + { + char link_target[PATH_BUFFER]; + + memcpy(link_target, reader.data + reader.position, + (size_t)stored_size); + link_target[stored_size] = '\0'; + reader.position += (size_t)stored_size; + make_parent_directories(target); + if (unlink(target) != 0 && errno != ENOENT) { + fail("cannot replace symlink '%s': %s", target, + strerror(errno)); + } + if (symlink(link_target, target) != 0) { + fail("cannot create symlink '%s': %s", target, + strerror(errno)); + } + } + } else if (type == ENTRY_FILE) { + int output; + + if (compression != COMP_NONE && compression != COMP_ZSTD) { + fail("file '%s' uses an unsupported compression", path); + } + if (compression == COMP_NONE && stored_size != original_size) { + fail("file '%s' has inconsistent sizes", path); + } + require_bytes(&reader, (size_t)stored_size); + payload = reader.data + reader.position; + reader.position += (size_t)stored_size; + + make_parent_directories(target); + output = open(target, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, + (mode_t)((mode & 0777U) | 0600U)); + if (output < 0) { + fail("cannot create '%s': %s", target, strerror(errno)); + } + if (compression == COMP_ZSTD) { + decompress_zstd(output, payload, (size_t)stored_size, + original_size, target); + } else { + write_all(output, payload, (size_t)stored_size, target); + } + if (fchmod(output, (mode_t)(mode & 0777U)) != 0) { + fail("cannot set permissions on '%s': %s", target, + strerror(errno)); + } + close(output); + } else { + fail("bundle archive contains an unknown entry type"); + } + free(target); + } + if (reader.position != reader.size) { + fail("bundle archive has trailing garbage"); + } + if (munmap((void *)archive, (size_t)trailer->archive_size) != 0) { + fail("cannot unmap bundle archive: %s", strerror(errno)); + } +} + +static void write_completion_marker(const char *directory, + const char *build_id) +{ + char *marker = join_path(directory, ".complete"); + int fd = open(marker, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0600); + + if (fd < 0) { + fail("cannot create completion marker '%s': %s", marker, + strerror(errno)); + } + write_all(fd, build_id, 32U, marker); + if (fsync(fd) != 0) { + fail("cannot flush completion marker '%s': %s", marker, + strerror(errno)); + } + close(fd); + free(marker); +} + +int main(int argc, char **argv) +{ + char *path = self_path(); + int bundle_fd = open(path, O_RDONLY | O_CLOEXEC); + struct stat status; + struct trailer trailer; + char *root; + char *directory; + char *lock_path; + int lock_fd; + char *binary; + + if (bundle_fd < 0) { + fail("cannot open '%s': %s", path, strerror(errno)); + } + if (fstat(bundle_fd, &status) != 0 || !S_ISREG(status.st_mode)) { + fail("'%s' is not a regular file", path); + } + trailer = read_trailer(bundle_fd, status.st_size); + + root = cache_root(); + make_directories(root, 0700); + directory = xmalloc(strlen(root) + 9U + 32U + 1U); + sprintf(directory, "%s/bundle-%s", root, trailer.build_id); + lock_path = xmalloc(strlen(root) + 6U + 32U + 5U + 1U); + sprintf(lock_path, "%s/lock-%s.lock", root, trailer.build_id); + + lock_fd = open(lock_path, O_RDWR | O_CREAT | O_CLOEXEC, 0600); + if (lock_fd < 0) { + fail("cannot create lock file '%s': %s", lock_path, strerror(errno)); + } + while (flock(lock_fd, LOCK_EX) != 0) { + if (errno != EINTR) { + fail("cannot lock '%s': %s", lock_path, strerror(errno)); + } + } + + if (!marker_matches(directory, trailer.build_id)) { + fprintf(stderr, "stemsep bundle: extracting runtime to %s\n", + directory); + make_directories(directory, 0700); + extract_archive(bundle_fd, &trailer, directory); + write_completion_marker(directory, trailer.build_id); + } + + close(lock_fd); + close(bundle_fd); + + if (argc > 1 && strcmp(argv[1], "--stemsep-extract") == 0) { + printf("%s\n", directory); + return 0; + } + if (argc > 1 && strcmp(argv[1], "--stemsep-licenses") == 0) { + printf("%s/licenses\n", directory); + return 0; + } + if (argc > 1 && strcmp(argv[1], "--stemsep-version") == 0) { + printf("stemsep bundle %s\n", trailer.build_id); + return 0; + } + + binary = join_path(directory, "stemsep"); + if (access(binary, X_OK) != 0) { + fail("extracted executable '%s' is not runnable: %s", binary, + strerror(errno)); + } + execv(binary, argv); + fail("cannot execute '%s': %s", binary, strerror(errno)); + return 127; +} diff --git a/packaging/licenses/DEMUCS-LICENSE.txt b/packaging/licenses/DEMUCS-LICENSE.txt new file mode 100644 index 0000000..b93be90 --- /dev/null +++ b/packaging/licenses/DEMUCS-LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) Meta Platforms, Inc. and affiliates. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packaging/licenses/HTDEMUCS-6S-ONNX-LICENSE.txt b/packaging/licenses/HTDEMUCS-6S-ONNX-LICENSE.txt new file mode 100644 index 0000000..51bf5d1 --- /dev/null +++ b/packaging/licenses/HTDEMUCS-6S-ONNX-LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 StemSplit + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packaging/mkbundle.py b/packaging/mkbundle.py new file mode 100755 index 0000000..714673b --- /dev/null +++ b/packaging/mkbundle.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +"""Build a single-file stemsep Linux bundle. + +The output layout is: + + [ELF launcher][zero padding to 4096][archive][64-byte trailer] + +The archive holds the application binary, the ONNX Runtime shared library, +the model, and license notices. Large entries are compressed with zstd. +""" + +from __future__ import annotations + +import argparse +import hashlib +import os +import shutil +import struct +import subprocess +import sys +import tempfile + +ARCHIVE_MAGIC = b"STEMSEPA" +ARCHIVE_VERSION = 1 +ENTRY_FILE = 0 +ENTRY_SYMLINK = 1 +COMP_NONE = 0 +COMP_ZSTD = 1 +TRAILER_MAGIC = b"STMSF001" +TRAILER_FORMAT = "<8sQQQ32s" +TRAILER_SIZE = struct.calcsize(TRAILER_FORMAT) +ALIGNMENT = 4096 +MAX_PATH = 4095 + + +def die(message: str) -> None: + print(f"mkbundle: error: {message}", file=sys.stderr) + raise SystemExit(1) + + +def check_archive_path(path: str) -> bytes: + if not path or path.startswith("/") or "\x00" in path: + die(f"unsafe archive path {path!r}") + if any(part in ("", ".", "..") for part in path.split("/")): + die(f"unsafe archive path {path!r}") + encoded = path.encode("utf-8") + if len(encoded) > MAX_PATH: + die(f"archive path is too long: {path!r}") + return encoded + + +def file_mode(source: str) -> int: + mode = os.stat(source).st_mode + return 0o755 if mode & 0o111 else 0o644 + + +def sha256_file(path: str) -> str: + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +class Archive: + def __init__(self, output, zstd_level: int, workdir: str) -> None: + self.output = output + self.zstd_level = zstd_level + self.workdir = workdir + self.digest = hashlib.sha256() + self.raw_bytes = 0 + self.stored_bytes = 0 + self.zstd_available = shutil.which("zstd") is not None + + def write(self, data: bytes) -> None: + self.digest.update(data) + self.output.write(data) + + def compress(self, source: str) -> tuple[int, int, str | None]: + """Return (original_size, stored_size, temporary_path_or_None).""" + original_size = os.path.getsize(source) + if ( + self.zstd_level <= 0 + or not self.zstd_available + or original_size < 1024 + ): + return original_size, original_size, None + + compressed = os.path.join(self.workdir, "entry.zst") + with open(compressed, "wb") as handle: + result = subprocess.run( + [ + "zstd", + "-q", + f"-{self.zstd_level}", + "-T0", + "-c", + "--", + source, + ], + stdout=handle, + stderr=subprocess.PIPE, + ) + if result.returncode != 0: + die( + "zstd failed for " + f"{source}: {result.stderr.decode(errors='replace').strip()}" + ) + stored_size = os.path.getsize(compressed) + if stored_size >= original_size - 32: + os.unlink(compressed) + return original_size, original_size, None + return original_size, stored_size, compressed + + def add_file(self, archive_path: str, source: str, mode: int, + compress: bool = True) -> None: + encoded_path = check_archive_path(archive_path) + if not os.path.isfile(source): + die(f"missing input file {source}") + original_size, stored_size, compressed = ( + self.compress(source) if compress else + (os.path.getsize(source), os.path.getsize(source), None) + ) + compression = COMP_ZSTD if compressed is not None else COMP_NONE + self.write( + struct.pack( + "<HBBHQQ", + len(encoded_path), + ENTRY_FILE, + compression, + mode, + original_size, + stored_size, + ) + ) + self.write(encoded_path) + source_path = compressed if compressed is not None else source + with open(source_path, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + self.write(chunk) + if compressed is not None: + os.unlink(compressed) + self.raw_bytes += original_size + self.stored_bytes += stored_size + + def add_symlink(self, archive_path: str, target: str) -> None: + encoded_path = check_archive_path(archive_path) + encoded_target = target.encode("utf-8") + if not encoded_target or len(encoded_target) >= MAX_PATH: + die(f"invalid symlink target for {archive_path!r}") + self.write( + struct.pack( + "<HBBHQQ", + len(encoded_path), + ENTRY_SYMLINK, + COMP_NONE, + 0o777, + len(encoded_target), + len(encoded_target), + ) + ) + self.write(encoded_path) + self.write(encoded_target) + self.raw_bytes += len(encoded_target) + self.stored_bytes += len(encoded_target) + + +def parse_assignments(values: list[str], label: str) -> list[tuple[str, str]]: + result = [] + for value in values: + if "=" not in value: + die(f"{label} expects ARCHIVE_PATH=VALUE, got {value!r}") + left, right = value.split("=", 1) + if not right: + die(f"{label} has an empty value in {value!r}") + result.append((left, right)) + return result + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--launcher", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--zstd-level", type=int, default=19) + parser.add_argument("--add", action="append", default=[], + metavar="ARCHIVE_PATH=SOURCE") + parser.add_argument("--symlink", action="append", default=[], + metavar="ARCHIVE_PATH=TARGET") + parser.add_argument("--checksum", action="append", default=[], + metavar="ARCHIVE_PATH=SHA256") + args = parser.parse_args() + + if not os.path.isfile(args.launcher): + die(f"launcher not found: {args.launcher}") + if args.zstd_level != 0 and shutil.which("zstd") is None: + print("mkbundle: warning: zstd not found, storing entries uncompressed", + file=sys.stderr) + args.zstd_level = 0 + + files = [(path, source) for path, source in + parse_assignments(args.add, "--add")] + links = parse_assignments(args.symlink, "--symlink") + checksums = { + path: expected.lower() + for path, expected in parse_assignments(args.checksum, "--checksum") + } + + for archive_path, expected in checksums.items(): + matches = [ + source for candidate, source in files if candidate == archive_path + ] + if not matches: + die(f"checksum given for unknown entry {archive_path!r}") + actual = sha256_file(matches[0]) + if actual != expected: + die( + f"checksum mismatch for {matches[0]}: " + f"expected {expected}, got {actual}" + ) + + os.makedirs(os.path.dirname(os.path.abspath(args.output)) or ".", + exist_ok=True) + temporary = args.output + ".tmp" + with tempfile.TemporaryDirectory(prefix="stemsep-mkbundle-") as workdir: + with open(temporary, "wb") as output: + with open(args.launcher, "rb") as launcher: + shutil.copyfileobj(launcher, output) + padding = (-output.tell()) % ALIGNMENT + output.write(b"\x00" * padding) + archive_offset = output.tell() + + archive = Archive(output, args.zstd_level, workdir) + archive.write(ARCHIVE_MAGIC) + archive.write(struct.pack("<II", ARCHIVE_VERSION, + len(files) + len(links))) + + for archive_path, source in files: + archive.add_file(archive_path, source, file_mode(source)) + for archive_path, target in links: + archive.add_symlink(archive_path, target) + + archive_size = output.tell() - archive_offset + build_id = archive.digest.hexdigest()[:32].encode("ascii") + output.write( + struct.pack( + TRAILER_FORMAT, + TRAILER_MAGIC, + archive_offset, + archive_size, + 0, + build_id, + ) + ) + os.chmod(temporary, 0o755) + os.replace(temporary, args.output) + + bundle_size = os.path.getsize(args.output) + ratio = 100.0 * archive.stored_bytes / archive.raw_bytes \ + if archive.raw_bytes else 0.0 + print( + f"mkbundle: wrote {args.output} " + f"({bundle_size} bytes, payload {archive.raw_bytes} -> " + f"{archive.stored_bytes} bytes, {ratio:.1f}%)" + ) + print(f"mkbundle: bundle sha256 {sha256_file(args.output)}") + print(f"mkbundle: build id {build_id.decode()}") + + +if __name__ == "__main__": + main() |
