#define _FILE_OFFSET_BITS 64 #define _POSIX_C_SOURCE 200809L #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "audio.h" #include "metronome.h" #include "playback.h" #include "wav_writer.h" #ifndef STEMSEP_TEST_FAKE_MODEL #include "onnxruntime_c_api.h" #endif #define APP_NAME "stemsep" #define MODEL_RELATIVE_PATH "models/htdemucs_6s.onnx" #define MODEL_INPUT_NAME "mix" #define MODEL_OUTPUT_NAME "stems" #define MODEL_SAMPLE_RATE UINT32_C(44100) #define MODEL_CHANNELS 2U #define MODEL_STEMS 6U #define MODEL_SEGMENT_FRAMES UINT64_C(343980) #define MODEL_OVERLAP_FRAMES UINT64_C(85995) #define MODEL_STRIDE_FRAMES UINT64_C(257985) #define RESAMPLE_PHASES 1024U #define RESAMPLE_BASE_TAPS 64U #define RESAMPLE_DOWNSAMPLE_TAPS_AT_48K 96U #define SOURCE_RING_FRAMES 4096U #define WAV_WRITE_BLOCK_FRAMES 4096U #define MIN_TRANSITION_WEIGHT 1.0e-8F #ifndef M_PI #define M_PI 3.14159265358979323846264338327950288 #endif _Static_assert(CHAR_BIT == 8, "stemsep requires 8-bit bytes"); _Static_assert(sizeof(float) == 4, "stemsep requires 32-bit float"); _Static_assert(FLT_RADIX == 2 && FLT_MANT_DIG == 24 && FLT_MAX_EXP == 128, "stemsep requires IEEE-754 binary32 float"); enum { EXIT_OK = 0, EXIT_FAILURE_GENERIC = 1, EXIT_USAGE = 2, EXIT_INPUT = 3, EXIT_MODEL = 4, EXIT_OUTPUT = 5, EXIT_INTERNAL = 6, EXIT_INTERRUPTED = 130 }; typedef struct { char message[768]; } Error; static volatile sig_atomic_t g_interrupted = 0; static void signal_handler(int signal_number) { (void)signal_number; g_interrupted = 1; } #if defined(__GNUC__) || defined(__clang__) __attribute__((format(printf, 2, 3))) #endif static bool set_error(Error *error, const char *format, ...) { va_list args; if (error != NULL) { va_start(args, format); (void)vsnprintf(error->message, sizeof(error->message), format, args); va_end(args); } return false; } static bool checked_add_u64(uint64_t left, uint64_t right, uint64_t *result) { if (UINT64_MAX - left < right) { return false; } *result = left + right; return true; } static bool checked_mul_u64(uint64_t left, uint64_t right, uint64_t *result) { if (left != 0U && right > UINT64_MAX / left) { return false; } *result = left * right; return true; } static bool checked_mul_size(size_t left, size_t right, size_t *result) { if (left != 0U && right > SIZE_MAX / left) { return false; } *result = left * right; return true; } static void *calloc_array(size_t count, size_t size, Error *error, const char *description) { size_t bytes; void *allocation; if (!checked_mul_size(count, size, &bytes)) { (void)set_error(error, "%s is too large", description); return NULL; } allocation = calloc(1U, bytes); if (allocation == NULL) { (void)set_error(error, "cannot allocate %zu bytes for %s: %s", bytes, description, strerror(errno)); } return allocation; } static char *duplicate_string(const char *source, Error *error) { size_t length = strlen(source); char *copy; if (length == SIZE_MAX) { (void)set_error(error, "path is too long"); return NULL; } copy = malloc(length + 1U); if (copy == NULL) { (void)set_error(error, "cannot allocate a path: %s", strerror(errno)); return NULL; } memcpy(copy, source, length + 1U); return copy; } static char *join_path(const char *directory, const char *name, Error *error) { size_t directory_length = strlen(directory); size_t name_length = strlen(name); bool needs_separator = directory_length > 0U && directory[directory_length - 1U] != '/'; size_t total; char *path; if (directory_length > SIZE_MAX - name_length - 2U) { (void)set_error(error, "path is too long"); return NULL; } total = directory_length + (needs_separator ? 1U : 0U) + name_length + 1U; path = malloc(total); if (path == NULL) { (void)set_error(error, "cannot allocate a path: %s", strerror(errno)); return NULL; } memcpy(path, directory, directory_length); if (needs_separator) { path[directory_length++] = '/'; } memcpy(path + directory_length, name, name_length); path[directory_length + name_length] = '\0'; return path; } static char *path_directory(const char *path, Error *error) { const char *slash = strrchr(path, '/'); size_t length; char *directory; if (slash == NULL) { return duplicate_string(".", error); } if (slash == path) { return duplicate_string("/", error); } length = (size_t)(slash - path); directory = malloc(length + 1U); if (directory == NULL) { (void)set_error(error, "cannot allocate a directory path: %s", strerror(errno)); return NULL; } memcpy(directory, path, length); directory[length] = '\0'; return directory; } static char *output_directory_name(const char *identifier, Error *error) { static const char prefix[] = "./"; static const char suffix[] = "_stems"; size_t length = strlen(identifier); size_t total; char *directory; if (length > SIZE_MAX - (sizeof(prefix) - 1U) - sizeof(suffix)) { (void)set_error(error, "music identifier is too long"); return NULL; } total = sizeof(prefix) - 1U + length + sizeof(suffix); directory = malloc(total); if (directory == NULL) { (void)set_error(error, "cannot allocate an output directory name: %s", strerror(errno)); return NULL; } (void)snprintf(directory, total, "./%s_stems", identifier); return directory; } static char *read_executable_path(Error *error) { size_t capacity = 256U; while (capacity <= (1U << 20)) { char *buffer = malloc(capacity); ssize_t length; if (buffer == NULL) { (void)set_error(error, "cannot allocate executable path: %s", strerror(errno)); return NULL; } length = readlink("/proc/self/exe", buffer, capacity - 1U); if (length < 0) { free(buffer); return NULL; } if ((size_t)length < capacity - 1U) { buffer[length] = '\0'; return buffer; } free(buffer); capacity *= 2U; } (void)set_error(error, "executable path is too long"); return NULL; } static char *model_path_from_executable(const char *argv0, Error *error) { const char *override = getenv("STEMSEP_MODEL"); char *executable = NULL; char *directory = NULL; char *model_path = NULL; if (override != NULL && override[0] != '\0') { return duplicate_string(override, error); } executable = read_executable_path(error); if (executable == NULL) { error->message[0] = '\0'; executable = duplicate_string(argv0, error); if (executable == NULL) { return NULL; } } directory = path_directory(executable, error); free(executable); if (directory == NULL) { return NULL; } model_path = join_path(directory, MODEL_RELATIVE_PATH, error); free(directory); return model_path; } static bool regular_readable_file(const char *path, Error *error, const char *description) { struct stat status; if (stat(path, &status) != 0) { return set_error(error, "cannot access %s '%s': %s", description, path, strerror(errno)); } if (!S_ISREG(status.st_mode)) { return set_error(error, "%s '%s' is not a regular file", description, path); } if (access(path, R_OK) != 0) { return set_error(error, "cannot read %s '%s': %s", description, path, strerror(errno)); } return true; } typedef struct { AudioSource *reader; float *frames; size_t capacity; size_t head; size_t count; uint64_t base_index; } SourceRing; static bool audio_open(AudioSource *source, const char *path, Error *error) { char message[768]; message[0] = '\0'; if (!audio_source_open(source, path, message, sizeof(message))) { return set_error(error, "%s", message[0] != '\0' ? message : "cannot open the input audio"); } return true; } static bool audio_read_frames(AudioSource *source, float *stereo, size_t requested, size_t *frames_read, Error *error) { char message[768]; message[0] = '\0'; if (!source->ops->read_frames(source, stereo, requested, frames_read, message, sizeof(message))) { return set_error(error, "%s", message[0] != '\0' ? message : "cannot read the input audio"); } return true; } static bool source_ring_init(SourceRing *ring, AudioSource *source, Error *error) { memset(ring, 0, sizeof(*ring)); ring->reader = source; ring->capacity = SOURCE_RING_FRAMES; ring->frames = calloc_array(ring->capacity * 2U, sizeof(float), error, "source audio ring"); return ring->frames != NULL; } static void source_ring_destroy(SourceRing *ring) { free(ring->frames); memset(ring, 0, sizeof(*ring)); } static void source_ring_discard_before(SourceRing *ring, uint64_t index) { while (ring->count > 0U && ring->base_index < index) { ring->head = (ring->head + 1U) % ring->capacity; --ring->count; ++ring->base_index; } } static bool source_ring_fill_until(SourceRing *ring, uint64_t wanted, Error *error) { while (ring->base_index + ring->count <= wanted) { size_t tail; size_t contiguous; size_t read_count = 0U; if (ring->base_index + ring->count >= ring->reader->total_frames) { return set_error(error, "resampler requested audio beyond the end of the input"); } if (ring->count == ring->capacity) { return set_error(error, "internal source audio ring overflow"); } tail = (ring->head + ring->count) % ring->capacity; contiguous = ring->capacity - tail; if (contiguous > ring->capacity - ring->count) { contiguous = ring->capacity - ring->count; } if (!audio_read_frames(ring->reader, ring->frames + tail * 2U, contiguous, &read_count, error)) { return false; } if (read_count == 0U) { return set_error(error, "unexpected end of decoded input audio"); } ring->count += read_count; } return true; } static bool source_ring_get(const SourceRing *ring, uint64_t index, float sample[2], Error *error) { uint64_t offset; size_t slot; if (index < ring->base_index) { return set_error(error, "internal resampler history underflow"); } offset = index - ring->base_index; if (offset >= ring->count) { return set_error(error, "internal resampler look-ahead underflow"); } slot = (ring->head + (size_t)offset) % ring->capacity; sample[0] = ring->frames[slot * 2U]; sample[1] = ring->frames[slot * 2U + 1U]; return true; } typedef struct { SourceRing source; uint32_t input_rate; uint64_t input_frames; uint64_t output_frames; uint64_t output_index; unsigned taps; unsigned radius; bool bypass; float *coefficients; } Resampler; static uint64_t reflect_frame_index(int64_t index, uint64_t frame_count) { int64_t period; int64_t reflected; if (frame_count <= 1U) { return 0U; } period = (int64_t)(2U * (frame_count - 1U)); reflected = index % period; if (reflected < 0) { reflected += period; } if ((uint64_t)reflected >= frame_count) { reflected = period - reflected; } return (uint64_t)reflected; } static double sinc_value(double value) { if (fabs(value) < 1.0e-12) { return 1.0; } return sin(M_PI * value) / (M_PI * value); } static bool resampler_build_coefficients(Resampler *resampler, Error *error) { unsigned phase; double rate_ratio = (double)MODEL_SAMPLE_RATE / (double)resampler->input_rate; double cutoff = 0.94 * (rate_ratio < 1.0 ? rate_ratio : 1.0); resampler->coefficients = calloc_array((size_t)RESAMPLE_PHASES * resampler->taps, sizeof(float), error, "resampling filter"); if (resampler->coefficients == NULL) { return false; } for (phase = 0U; phase < RESAMPLE_PHASES; ++phase) { double fraction = (double)phase / (double)RESAMPLE_PHASES; double sum = 0.0; unsigned tap; for (tap = 0U; tap < resampler->taps; ++tap) { int offset = (int)tap - ((int)resampler->radius - 1); double x = (double)offset - fraction; double window; double weight; if (fabs(x) >= (double)resampler->radius) { window = 0.0; } else { window = 0.42 + 0.5 * cos(M_PI * x / (double)resampler->radius) + 0.08 * cos(2.0 * M_PI * x / (double)resampler->radius); } weight = cutoff * sinc_value(cutoff * x) * window; resampler->coefficients[(size_t)phase * resampler->taps + tap] = (float)weight; sum += weight; } if (fabs(sum) < 1.0e-12) { return set_error(error, "cannot normalize the resampling filter"); } for (tap = 0U; tap < resampler->taps; ++tap) { size_t index = (size_t)phase * resampler->taps + tap; resampler->coefficients[index] = (float)(resampler->coefficients[index] / sum); } } return true; } static bool resampler_init(Resampler *resampler, AudioSource *source, Error *error) { uint64_t scaled; uint64_t rounded; memset(resampler, 0, sizeof(*resampler)); resampler->input_rate = source->sample_rate; resampler->input_frames = source->total_frames; resampler->bypass = source->sample_rate == MODEL_SAMPLE_RATE; resampler->taps = RESAMPLE_BASE_TAPS; if (source->sample_rate > MODEL_SAMPLE_RATE) { uint64_t scaled_taps = (uint64_t)RESAMPLE_DOWNSAMPLE_TAPS_AT_48K * source->sample_rate; uint64_t rounded_taps = (scaled_taps + UINT64_C(47999)) / UINT64_C(48000); if ((rounded_taps & 1U) != 0U) { ++rounded_taps; } if (rounded_taps > UINT_MAX || rounded_taps + 1U > SOURCE_RING_FRAMES) { return set_error(error, "unsupported resampling filter size"); } resampler->taps = (unsigned)rounded_taps; } resampler->radius = resampler->taps / 2U; if (!checked_mul_u64(source->total_frames, MODEL_SAMPLE_RATE, &scaled) || !checked_add_u64(scaled, source->sample_rate / 2U, &rounded)) { return set_error(error, "resampled audio duration is too large"); } resampler->output_frames = rounded / source->sample_rate; if (resampler->output_frames == 0U) { resampler->output_frames = 1U; } if (!source_ring_init(&resampler->source, source, error)) { return false; } if (!resampler->bypass && !resampler_build_coefficients(resampler, error)) { source_ring_destroy(&resampler->source); return false; } return true; } static void resampler_destroy(Resampler *resampler) { source_ring_destroy(&resampler->source); free(resampler->coefficients); memset(resampler, 0, sizeof(*resampler)); } static bool resampler_next(Resampler *resampler, float output[2], Error *error) { uint64_t source_numerator; uint64_t base; uint64_t remainder; uint64_t phase_numerator; unsigned phase; if (resampler->output_index >= resampler->output_frames) { return set_error(error, "internal resampler output overrun"); } if (resampler->bypass) { uint64_t index = resampler->output_index; source_ring_discard_before(&resampler->source, index); if (!source_ring_fill_until(&resampler->source, index, error) || !source_ring_get(&resampler->source, index, output, error)) { return false; } ++resampler->output_index; return true; } if (!checked_mul_u64(resampler->output_index, resampler->input_rate, &source_numerator)) { return set_error(error, "resampler position overflow"); } base = source_numerator / MODEL_SAMPLE_RATE; remainder = source_numerator % MODEL_SAMPLE_RATE; phase_numerator = remainder * RESAMPLE_PHASES + MODEL_SAMPLE_RATE / 2U; phase = (unsigned)(phase_numerator / MODEL_SAMPLE_RATE); if (phase == RESAMPLE_PHASES) { phase = 0U; ++base; } if (base >= resampler->input_frames) { base = resampler->input_frames - 1U; phase = 0U; } { uint64_t keep_from = base > resampler->radius ? base - resampler->radius : 0U; uint64_t need_until = base + resampler->radius; double left = 0.0; double right = 0.0; unsigned tap; if (need_until >= resampler->input_frames) { need_until = resampler->input_frames - 1U; } source_ring_discard_before(&resampler->source, keep_from); if (!source_ring_fill_until(&resampler->source, need_until, error)) { return false; } for (tap = 0U; tap < resampler->taps; ++tap) { int offset = (int)tap - ((int)resampler->radius - 1); uint64_t source_index = reflect_frame_index((int64_t)base + offset, resampler->input_frames); float sample[2]; float coefficient = resampler->coefficients[(size_t)phase * resampler->taps + tap]; if (!source_ring_get(&resampler->source, source_index, sample, error)) { return false; } left += (double)sample[0] * coefficient; right += (double)sample[1] * coefficient; } output[0] = (float)left; output[1] = (float)right; } if (!isfinite(output[0]) || !isfinite(output[1])) { return set_error(error, "resampler produced a non-finite sample"); } ++resampler->output_index; return true; } typedef struct { WavWriter *writer; char *partial_path; char *final_path; bool partial_created; bool renamed; } OutputFile; typedef struct { char *directory; OutputFile files[MODEL_STEMS]; bool directory_created; bool committed; } OutputSet; static const char *const STEM_NAMES[MODEL_STEMS] = { "drums", "bass", "other", "vocals", "guitar", "piano" }; static bool output_file_open(OutputFile *file, const char *directory, const char *stem_name, uint64_t expected_frames, Error *error) { size_t name_length = strlen(stem_name); char *final_name = NULL; char *partial_name = NULL; char message[768]; FILE *stream; int descriptor; int flags = O_WRONLY | O_CREAT | O_EXCL; memset(file, 0, sizeof(*file)); if (name_length > SIZE_MAX - sizeof(".wav.part")) { return set_error(error, "output filename is too long"); } final_name = malloc(name_length + sizeof(".wav")); partial_name = malloc(name_length + sizeof(".wav.part")); if (final_name == NULL || partial_name == NULL) { free(final_name); free(partial_name); return set_error(error, "cannot allocate an output filename: %s", strerror(errno)); } (void)snprintf(final_name, name_length + sizeof(".wav"), "%s.wav", stem_name); (void)snprintf(partial_name, name_length + sizeof(".wav.part"), "%s.wav.part", stem_name); file->final_path = join_path(directory, final_name, error); file->partial_path = join_path(directory, partial_name, error); free(final_name); free(partial_name); if (file->final_path == NULL || file->partial_path == NULL) { return false; } #ifdef O_CLOEXEC flags |= O_CLOEXEC; #endif #ifdef O_NOFOLLOW flags |= O_NOFOLLOW; #endif descriptor = open(file->partial_path, flags, 0644); if (descriptor < 0) { return set_error(error, "cannot create output '%s': %s", file->partial_path, strerror(errno)); } file->partial_created = true; stream = fdopen(descriptor, "wb"); if (stream == NULL) { int saved_errno = errno; (void)close(descriptor); (void)unlink(file->partial_path); file->partial_created = false; return set_error(error, "cannot open output stream '%s': %s", file->partial_path, strerror(saved_errno)); } message[0] = '\0'; file->writer = wav_writer_open(stream, file->partial_path, MODEL_SAMPLE_RATE, MODEL_CHANNELS, expected_frames, message, sizeof(message)); if (file->writer == NULL) { (void)unlink(file->partial_path); file->partial_created = false; return set_error(error, "%s", message[0] != '\0' ? message : "cannot create output WAV"); } return true; } static bool output_file_write(OutputFile *file, const float *interleaved, size_t frame_count, Error *error) { char message[768]; message[0] = '\0'; if (!wav_writer_write(file->writer, interleaved, frame_count, message, sizeof(message))) { return set_error(error, "%s", message[0] != '\0' ? message : "cannot write the output WAV"); } return true; } static bool output_file_finish(OutputFile *file, Error *error) { char message[768]; bool ok; message[0] = '\0'; ok = wav_writer_finish(file->writer, message, sizeof(message)); file->writer = NULL; if (!ok) { return set_error(error, "%s", message[0] != '\0' ? message : "cannot finish the output WAV"); } return true; } static void output_file_cleanup(OutputFile *file, bool remove_files) { if (file->writer != NULL) { wav_writer_abort(file->writer); file->writer = NULL; } if (remove_files) { if (file->partial_created && file->partial_path != NULL) { (void)unlink(file->partial_path); } if (file->renamed && file->final_path != NULL) { (void)unlink(file->final_path); } } free(file->partial_path); free(file->final_path); memset(file, 0, sizeof(*file)); } static bool output_set_create(OutputSet *outputs, const char *directory, const char *identifier, uint64_t expected_frames, Error *error) { unsigned stem; memset(outputs, 0, sizeof(*outputs)); outputs->directory = duplicate_string(directory, error); if (outputs->directory == NULL) { return false; } if (mkdir(directory, 0755) != 0) { if (errno == EEXIST) { return set_error(error, "output directory '%s' already exists", directory); } return set_error(error, "cannot create output directory '%s': %s", directory, strerror(errno)); } outputs->directory_created = true; for (stem = 0U; stem < MODEL_STEMS; ++stem) { size_t identifier_length = strlen(identifier); size_t stem_length = strlen(STEM_NAMES[stem]); char *prefixed_name; if (identifier_length > SIZE_MAX - stem_length - 2U) { return set_error(error, "output filename is too long"); } prefixed_name = malloc(identifier_length + stem_length + 2U); if (prefixed_name == NULL) { return set_error(error, "cannot allocate an output filename: %s", strerror(errno)); } (void)snprintf(prefixed_name, identifier_length + stem_length + 2U, "%s_%s", identifier, STEM_NAMES[stem]); if (!output_file_open(&outputs->files[stem], directory, prefixed_name, expected_frames, error)) { free(prefixed_name); return false; } free(prefixed_name); } return true; } static bool output_set_commit(OutputSet *outputs, Error *error) { unsigned stem; for (stem = 0U; stem < MODEL_STEMS; ++stem) { if (!output_file_finish(&outputs->files[stem], error)) { return false; } } for (stem = 0U; stem < MODEL_STEMS; ++stem) { if (rename(outputs->files[stem].partial_path, outputs->files[stem].final_path) != 0) { return set_error(error, "cannot finalize output '%s': %s", outputs->files[stem].final_path, strerror(errno)); } outputs->files[stem].partial_created = false; outputs->files[stem].renamed = true; } outputs->committed = true; return true; } static void output_set_cleanup(OutputSet *outputs) { unsigned stem; bool remove_files = !outputs->committed; for (stem = 0U; stem < MODEL_STEMS; ++stem) { output_file_cleanup(&outputs->files[stem], remove_files); } if (remove_files && outputs->directory_created && outputs->directory != NULL) { (void)rmdir(outputs->directory); } free(outputs->directory); memset(outputs, 0, sizeof(*outputs)); } typedef struct { #ifndef STEMSEP_TEST_FAKE_MODEL const OrtApi *api; OrtEnv *environment; OrtSessionOptions *session_options; OrtSession *session; OrtMemoryInfo *memory_info; OrtValue *input_value; #else float *fake_output; #endif } Model; typedef struct { const float *samples; #ifndef STEMSEP_TEST_FAKE_MODEL OrtValue *owned_value; #endif } ModelResult; #ifndef STEMSEP_TEST_FAKE_MODEL static bool ort_ok(Model *model, OrtStatus *status, Error *error, const char *operation) { const char *message; if (status == NULL) { return true; } message = model->api->GetErrorMessage(status); (void)set_error(error, "ONNX Runtime %s failed: %s", operation, message != NULL ? message : "unknown error"); model->api->ReleaseStatus(status); return false; } static bool model_validate_contract(Model *model, bool input, const int64_t *expected_dimensions, size_t expected_rank, Error *error) { OrtTypeInfo *type_info = NULL; const OrtTensorTypeAndShapeInfo *tensor_info = NULL; enum ONNXTensorElementDataType element_type; int64_t dimensions[4] = {0, 0, 0, 0}; size_t rank = 0U; size_t index; OrtStatus *status; bool ok = false; status = input ? model->api->SessionGetInputTypeInfo(model->session, 0U, &type_info) : model->api->SessionGetOutputTypeInfo(model->session, 0U, &type_info); if (!ort_ok(model, status, error, input ? "input inspection" : "output inspection")) { goto cleanup; } if (!ort_ok(model, model->api->CastTypeInfoToTensorInfo(type_info, &tensor_info), error, "tensor type inspection") || tensor_info == NULL) { if (tensor_info == NULL && error->message[0] == '\0') { (void)set_error(error, "model %s is not a tensor", input ? "input" : "output"); } goto cleanup; } if (!ort_ok(model, model->api->GetTensorElementType(tensor_info, &element_type), error, "tensor element type inspection") || !ort_ok(model, model->api->GetDimensionsCount(tensor_info, &rank), error, "tensor rank inspection")) { goto cleanup; } if (element_type != ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT || rank != expected_rank || rank > sizeof(dimensions) / sizeof(dimensions[0])) { (void)set_error(error, "model %s contract is incompatible: expected float32 rank %zu", input ? "input" : "output", expected_rank); goto cleanup; } if (!ort_ok(model, model->api->GetDimensions(tensor_info, dimensions, rank), error, "tensor dimension inspection")) { goto cleanup; } for (index = 0U; index < rank; ++index) { if (dimensions[index] != expected_dimensions[index]) { (void)set_error(error, "model %s dimension %zu is incompatible: expected %" PRId64 ", found %" PRId64, input ? "input" : "output", index, expected_dimensions[index], dimensions[index]); goto cleanup; } } ok = true; cleanup: if (type_info != NULL) { model->api->ReleaseTypeInfo(type_info); } return ok; } static bool model_validate_name(Model *model, bool input, const char *expected_name, Error *error) { OrtAllocator *allocator = NULL; char *actual_name = NULL; OrtStatus *status; bool ok = false; if (!ort_ok(model, model->api->GetAllocatorWithDefaultOptions(&allocator), error, "default-allocator access") || allocator == NULL) { if (allocator == NULL && error->message[0] == '\0') { (void)set_error(error, "ONNX Runtime returned no default allocator"); } return false; } status = input ? model->api->SessionGetInputName(model->session, 0U, allocator, &actual_name) : model->api->SessionGetOutputName(model->session, 0U, allocator, &actual_name); if (!ort_ok(model, status, error, input ? "input-name inspection" : "output-name inspection")) { goto cleanup; } if (actual_name == NULL || strcmp(actual_name, expected_name) != 0) { (void)set_error(error, "model %s name is incompatible: expected '%s', found '%s'", input ? "input" : "output", expected_name, actual_name != NULL ? actual_name : "(null)"); goto cleanup; } ok = true; cleanup: if (actual_name != NULL) { allocator->Free(allocator, actual_name); } return ok; } #endif static bool model_init(Model *model, const char *model_path, float *input_buffer, Error *error) { memset(model, 0, sizeof(*model)); #ifdef STEMSEP_TEST_FAKE_MODEL (void)model_path; (void)input_buffer; model->fake_output = calloc_array((size_t)(MODEL_STEMS * MODEL_CHANNELS * MODEL_SEGMENT_FRAMES), sizeof(float), error, "fake model output"); return model->fake_output != NULL; #else const OrtApiBase *base = OrtGetApiBase(); int64_t input_dimensions[3] = { 1, (int64_t)MODEL_CHANNELS, (int64_t)MODEL_SEGMENT_FRAMES }; int64_t output_dimensions[4] = { 1, (int64_t)MODEL_STEMS, (int64_t)MODEL_CHANNELS, (int64_t)MODEL_SEGMENT_FRAMES }; size_t input_count = 0U; size_t output_count = 0U; size_t input_bytes = (size_t)(MODEL_CHANNELS * MODEL_SEGMENT_FRAMES) * sizeof(float); if (base == NULL) { return set_error(error, "ONNX Runtime returned no API base"); } model->api = base->GetApi(ORT_API_VERSION); if (model->api == NULL) { return set_error(error, "ONNX Runtime API %d is unavailable (runtime reports %s)", ORT_API_VERSION, base->GetVersionString()); } if (!ort_ok(model, model->api->CreateEnv(ORT_LOGGING_LEVEL_WARNING, APP_NAME, &model->environment), error, "environment creation") || !ort_ok(model, model->api->CreateSessionOptions(&model->session_options), error, "session-options creation") || !ort_ok(model, model->api->SetSessionGraphOptimizationLevel(model->session_options, ORT_ENABLE_ALL), error, "graph optimization setup") || !ort_ok(model, model->api->CreateSession(model->environment, model_path, model->session_options, &model->session), error, "model loading") || !ort_ok(model, model->api->SessionGetInputCount(model->session, &input_count), error, "input-count inspection") || !ort_ok(model, model->api->SessionGetOutputCount(model->session, &output_count), error, "output-count inspection")) { return false; } if (input_count != 1U || output_count != 1U) { return set_error(error, "model contract is incompatible: expected one input and one output"); } if (!model_validate_name(model, true, MODEL_INPUT_NAME, error) || !model_validate_name(model, false, MODEL_OUTPUT_NAME, error) || !model_validate_contract(model, true, input_dimensions, 3U, error) || !model_validate_contract(model, false, output_dimensions, 4U, error) || !ort_ok(model, model->api->CreateCpuMemoryInfo(OrtArenaAllocator, OrtMemTypeDefault, &model->memory_info), error, "CPU memory-info creation") || !ort_ok(model, model->api->CreateTensorWithDataAsOrtValue( model->memory_info, input_buffer, input_bytes, input_dimensions, 3U, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, &model->input_value), error, "input-tensor creation")) { return false; } fprintf(stderr, "ONNX Runtime %s, CPU execution provider\n", base->GetVersionString()); return true; #endif } static void model_destroy(Model *model) { #ifdef STEMSEP_TEST_FAKE_MODEL free(model->fake_output); #else if (model->api != NULL) { if (model->input_value != NULL) { model->api->ReleaseValue(model->input_value); } if (model->memory_info != NULL) { model->api->ReleaseMemoryInfo(model->memory_info); } if (model->session != NULL) { model->api->ReleaseSession(model->session); } if (model->session_options != NULL) { model->api->ReleaseSessionOptions(model->session_options); } if (model->environment != NULL) { model->api->ReleaseEnv(model->environment); } } #endif memset(model, 0, sizeof(*model)); } static bool model_run(Model *model, const float *input_buffer, ModelResult *result, Error *error) { memset(result, 0, sizeof(*result)); #ifdef STEMSEP_TEST_FAKE_MODEL unsigned stem; unsigned channel; uint64_t frame; (void)error; for (stem = 0U; stem < MODEL_STEMS; ++stem) { float scale = (float)(stem + 1U) / (float)MODEL_STEMS; for (channel = 0U; channel < MODEL_CHANNELS; ++channel) { size_t output_offset = (size_t)((stem * MODEL_CHANNELS + channel) * MODEL_SEGMENT_FRAMES); size_t input_offset = (size_t)(channel * MODEL_SEGMENT_FRAMES); for (frame = 0U; frame < MODEL_SEGMENT_FRAMES; ++frame) { model->fake_output[output_offset + (size_t)frame] = input_buffer[input_offset + (size_t)frame] * scale; } } } result->samples = model->fake_output; return true; #else const char *input_names[1] = {MODEL_INPUT_NAME}; const char *output_names[1] = {MODEL_OUTPUT_NAME}; const OrtValue *input_values[1] = {model->input_value}; OrtValue *output_values[1] = {NULL}; OrtStatus *status; void *raw_data = NULL; int is_tensor = 0; (void)input_buffer; status = model->api->Run(model->session, NULL, input_names, input_values, 1U, output_names, 1U, output_values); if (status != NULL) { if (output_values[0] != NULL) { model->api->ReleaseValue(output_values[0]); } (void)ort_ok(model, status, error, "inference"); return false; } if (output_values[0] == NULL) { return set_error(error, "ONNX Runtime returned no output tensor"); } result->owned_value = output_values[0]; if (!ort_ok(model, model->api->IsTensor(result->owned_value, &is_tensor), error, "output tensor validation")) { model->api->ReleaseValue(result->owned_value); result->owned_value = NULL; return false; } if (!is_tensor) { model->api->ReleaseValue(result->owned_value); result->owned_value = NULL; return set_error(error, "ONNX Runtime output is not a tensor"); } if (!ort_ok(model, model->api->GetTensorMutableData(result->owned_value, &raw_data), error, "output tensor access")) { model->api->ReleaseValue(result->owned_value); result->owned_value = NULL; return false; } if (raw_data == NULL) { model->api->ReleaseValue(result->owned_value); result->owned_value = NULL; return set_error(error, "ONNX Runtime returned a null output buffer"); } result->samples = raw_data; return true; #endif } static void model_result_release(Model *model, ModelResult *result) { #ifndef STEMSEP_TEST_FAKE_MODEL if (result->owned_value != NULL) { model->api->ReleaseValue(result->owned_value); } #else (void)model; #endif memset(result, 0, sizeof(*result)); } static bool build_transition_window(float *window, Error *error) { uint64_t frame; if (MODEL_OVERLAP_FRAMES < 2U || MODEL_OVERLAP_FRAMES * 2U > MODEL_SEGMENT_FRAMES) { return set_error(error, "invalid compiled overlap constants"); } for (frame = 0U; frame < MODEL_SEGMENT_FRAMES; ++frame) { window[frame] = 1.0F; } for (frame = 0U; frame < MODEL_OVERLAP_FRAMES; ++frame) { float fade = (float)frame / (float)(MODEL_OVERLAP_FRAMES - 1U); if (fade < MIN_TRANSITION_WEIGHT) { fade = MIN_TRANSITION_WEIGHT; } window[frame] = fade; window[MODEL_SEGMENT_FRAMES - 1U - frame] = fade; } return true; } static bool fill_model_input(float *input, Resampler *resampler, uint64_t chunk_start, uint64_t target_end, uint64_t *loaded_frames, Error *error) { while (*loaded_frames < target_end) { uint64_t destination = *loaded_frames - chunk_start; float sample[2]; if (destination >= MODEL_SEGMENT_FRAMES) { return set_error(error, "internal model-input assembly overflow"); } if (!resampler_next(resampler, sample, error)) { return false; } input[destination] = sample[0]; input[MODEL_SEGMENT_FRAMES + destination] = sample[1]; ++*loaded_frames; } return true; } static bool accumulate_model_output(float *accumulator, float *weight, const float *model_output, const float *window, uint64_t valid_frames, Error *error) { uint64_t frame; unsigned plane; for (frame = 0U; frame < valid_frames; ++frame) { weight[frame] += window[frame]; } for (plane = 0U; plane < MODEL_STEMS * MODEL_CHANNELS; ++plane) { size_t base = (size_t)(plane * MODEL_SEGMENT_FRAMES); for (frame = 0U; frame < valid_frames; ++frame) { float sample = model_output[base + (size_t)frame]; if (!isfinite(sample)) { return set_error(error, "model produced a NaN or infinite sample"); } accumulator[base + (size_t)frame] += sample * window[frame]; } } return true; } static bool write_finalized_audio(OutputSet *outputs, const float *accumulator, const float *weight, uint64_t frame_count, Error *error) { float interleaved[WAV_WRITE_BLOCK_FRAMES * MODEL_CHANNELS]; unsigned stem; for (stem = 0U; stem < MODEL_STEMS; ++stem) { uint64_t offset = 0U; while (offset < frame_count) { size_t block = (size_t)(frame_count - offset); size_t frame; if (block > WAV_WRITE_BLOCK_FRAMES) { block = WAV_WRITE_BLOCK_FRAMES; } for (frame = 0U; frame < block; ++frame) { uint64_t source_frame = offset + frame; float divisor = weight[source_frame]; unsigned channel; if (divisor < MIN_TRANSITION_WEIGHT) { divisor = MIN_TRANSITION_WEIGHT; } for (channel = 0U; channel < MODEL_CHANNELS; ++channel) { size_t plane = stem * MODEL_CHANNELS + channel; float value = accumulator[plane * MODEL_SEGMENT_FRAMES + (size_t)source_frame] / divisor; if (!isfinite(value)) { return set_error(error, "overlap-add produced a non-finite %s sample", STEM_NAMES[stem]); } interleaved[frame * MODEL_CHANNELS + channel] = value; } } if (!output_file_write(&outputs->files[stem], interleaved, block, error)) { return false; } offset += block; } } return true; } static void shift_overlap_buffers(float *input, float *accumulator, float *weight) { unsigned plane; for (plane = 0U; plane < MODEL_CHANNELS; ++plane) { float *channel = input + plane * MODEL_SEGMENT_FRAMES; memmove(channel, channel + MODEL_STRIDE_FRAMES, (size_t)MODEL_OVERLAP_FRAMES * sizeof(float)); memset(channel + MODEL_OVERLAP_FRAMES, 0, (size_t)MODEL_STRIDE_FRAMES * sizeof(float)); } for (plane = 0U; plane < MODEL_STEMS * MODEL_CHANNELS; ++plane) { float *values = accumulator + plane * MODEL_SEGMENT_FRAMES; memmove(values, values + MODEL_STRIDE_FRAMES, (size_t)MODEL_OVERLAP_FRAMES * sizeof(float)); memset(values + MODEL_OVERLAP_FRAMES, 0, (size_t)MODEL_STRIDE_FRAMES * sizeof(float)); } memmove(weight, weight + MODEL_STRIDE_FRAMES, (size_t)MODEL_OVERLAP_FRAMES * sizeof(float)); memset(weight + MODEL_OVERLAP_FRAMES, 0, (size_t)MODEL_STRIDE_FRAMES * sizeof(float)); } static bool separate_audio(Model *model, Resampler *resampler, OutputSet *outputs, float *input, float *accumulator, float *weight, const float *window, int *failure_code, Error *error) { uint64_t total_frames = resampler->output_frames; uint64_t loaded_frames = 0U; uint64_t chunk_count; uint64_t chunk; uint64_t first_target = total_frames < MODEL_SEGMENT_FRAMES ? total_frames : MODEL_SEGMENT_FRAMES; chunk_count = (total_frames + MODEL_STRIDE_FRAMES - 1U) / MODEL_STRIDE_FRAMES; if (chunk_count == 0U) { chunk_count = 1U; } if (!fill_model_input(input, resampler, 0U, first_target, &loaded_frames, error)) { *failure_code = EXIT_INPUT; return false; } for (chunk = 0U; chunk < chunk_count; ++chunk) { uint64_t start = chunk * MODEL_STRIDE_FRAMES; uint64_t remaining = total_frames - start; uint64_t valid_frames = remaining < MODEL_SEGMENT_FRAMES ? remaining : MODEL_SEGMENT_FRAMES; uint64_t finalized_frames = chunk + 1U == chunk_count ? valid_frames : MODEL_STRIDE_FRAMES; ModelResult result; if (g_interrupted) { *failure_code = EXIT_INTERRUPTED; return set_error(error, "interrupted"); } fprintf(stderr, "Separating chunk %" PRIu64 "/%" PRIu64 " (%5.1f%%)\n", chunk + 1U, chunk_count, 100.0 * (double)(chunk + 1U) / (double)chunk_count); if (!model_run(model, input, &result, error)) { *failure_code = EXIT_MODEL; return false; } if (!accumulate_model_output(accumulator, weight, result.samples, window, valid_frames, error)) { model_result_release(model, &result); *failure_code = EXIT_MODEL; return false; } model_result_release(model, &result); if (!write_finalized_audio(outputs, accumulator, weight, finalized_frames, error)) { *failure_code = EXIT_OUTPUT; return false; } if (chunk + 1U < chunk_count) { uint64_t next_start = start + MODEL_STRIDE_FRAMES; uint64_t next_end; shift_overlap_buffers(input, accumulator, weight); if (!checked_add_u64(next_start, MODEL_SEGMENT_FRAMES, &next_end) || next_end > total_frames) { next_end = total_frames; } if (!fill_model_input(input, resampler, next_start, next_end, &loaded_frames, error)) { *failure_code = EXIT_INPUT; return false; } } } return true; } static bool install_signal_handlers(Error *error) { struct sigaction action; memset(&action, 0, sizeof(action)); action.sa_handler = signal_handler; if (sigemptyset(&action.sa_mask) != 0 || sigaction(SIGINT, &action, NULL) != 0 || sigaction(SIGTERM, &action, NULL) != 0) { return set_error(error, "cannot install signal handlers: %s", strerror(errno)); } return true; } static void print_usage(const char *program) { fprintf(stderr, "Usage:\n" " %s -i -m \n" " %s -m -p \n" " %s --bpm [-t ]\n", program, program, program); } typedef struct { const char *input_path; const char *identifier; const char *play_selection; const char *bpm; const char *time_signature; } CommandLine; static bool parse_command_line(int argc, char **argv, CommandLine *command, Error *error) { int index; memset(command, 0, sizeof(*command)); for (index = 1; index < argc; ++index) { const char *flag = argv[index]; const char **destination; if (strcmp(flag, "-i") == 0 || strcmp(flag, "--input") == 0 || strcmp(flag, "-wav") == 0) { destination = &command->input_path; } else if (strcmp(flag, "-m") == 0) { destination = &command->identifier; } else if (strcmp(flag, "-p") == 0) { destination = &command->play_selection; } else if (strcmp(flag, "--bpm") == 0) { destination = &command->bpm; } else if (strcmp(flag, "-t") == 0 || strcmp(flag, "--time-signature") == 0) { destination = &command->time_signature; } else { return set_error(error, "unknown option '%s'", flag); } if (*destination != NULL) { return set_error(error, "option '%s' was provided more than once", flag); } if (index + 1 >= argc || argv[index + 1][0] == '\0') { return set_error(error, "option '%s' requires a value", flag); } *destination = argv[++index]; } if (command->bpm != NULL) { if (command->input_path != NULL || command->identifier != NULL || command->play_selection != NULL) { return set_error(error, "option '--bpm' cannot be combined with '-i', '-m' or '-p'"); } return true; } if (command->time_signature != NULL) { return set_error(error, "option '-t' requires '--bpm'"); } if (command->identifier == NULL) { return set_error(error, "option '-m' is required"); } if ((command->input_path == NULL) == (command->play_selection == NULL)) { return set_error(error, "provide exactly one of '-i' or '-p'"); } return stem_identifier_validate(command->identifier, error->message, sizeof(error->message)); } int main(int argc, char **argv) { Error error = {{0}}; AudioSource source; Resampler resampler; Model model; OutputSet outputs; CommandLine command; char *model_path = NULL; char *output_directory = NULL; float *input = NULL; float *accumulator = NULL; float *weight = NULL; float *window = NULL; uint64_t output_bytes; int exit_code = EXIT_FAILURE_GENERIC; memset(&source, 0, sizeof(source)); memset(&resampler, 0, sizeof(resampler)); memset(&model, 0, sizeof(model)); memset(&outputs, 0, sizeof(outputs)); if (!parse_command_line(argc, argv, &command, &error)) { if (error.message[0] != '\0') { fprintf(stderr, "error: %s\n", error.message); } print_usage(argv[0]); return EXIT_USAGE; } if (command.bpm != NULL) { double bpm = 0.0; unsigned beats = METRONOME_DEFAULT_BEATS; if (!metronome_bpm_validate(command.bpm, &bpm, error.message, sizeof(error.message)) || (command.time_signature != NULL && !metronome_beats_validate(command.time_signature, &beats, error.message, sizeof(error.message)))) { fprintf(stderr, "error: %s\n", error.message); print_usage(argv[0]); return EXIT_USAGE; } return metronome_run(bpm, beats); } if (command.play_selection != NULL) { int status = playback_run(command.identifier, command.play_selection); if (status == EXIT_USAGE) { print_usage(argv[0]); } return status; } if (!install_signal_handlers(&error)) { exit_code = EXIT_INTERNAL; goto cleanup; } if (!regular_readable_file(command.input_path, &error, "input audio") || !audio_open(&source, command.input_path, &error)) { exit_code = EXIT_INPUT; goto cleanup; } if (!resampler_init(&resampler, &source, &error)) { exit_code = EXIT_INPUT; goto cleanup; } if (!checked_mul_u64(resampler.output_frames, MODEL_CHANNELS * sizeof(float), &output_bytes) || output_bytes > UINT32_MAX - 36U) { (void)set_error(&error, "resampled audio is too long for standard RIFF WAV output"); exit_code = EXIT_OUTPUT; goto cleanup; } model_path = model_path_from_executable(argv[0], &error); if (model_path == NULL) { exit_code = EXIT_INTERNAL; goto cleanup; } #ifndef STEMSEP_TEST_FAKE_MODEL if (!regular_readable_file(model_path, &error, "ONNX model")) { exit_code = EXIT_MODEL; goto cleanup; } #endif output_directory = output_directory_name(command.identifier, &error); if (output_directory == NULL) { exit_code = EXIT_INTERNAL; goto cleanup; } input = calloc_array((size_t)(MODEL_CHANNELS * MODEL_SEGMENT_FRAMES), sizeof(float), &error, "model input"); accumulator = calloc_array((size_t)(MODEL_STEMS * MODEL_CHANNELS * MODEL_SEGMENT_FRAMES), sizeof(float), &error, "overlap accumulator"); weight = calloc_array((size_t)MODEL_SEGMENT_FRAMES, sizeof(float), &error, "overlap weights"); window = calloc_array((size_t)MODEL_SEGMENT_FRAMES, sizeof(float), &error, "transition window"); if (input == NULL || accumulator == NULL || weight == NULL || window == NULL || !build_transition_window(window, &error)) { exit_code = EXIT_INTERNAL; goto cleanup; } fprintf(stderr, "Input: %s, %u channel%s, %" PRIu32 " Hz, %" PRIu64 " frames -> stereo %u Hz, %" PRIu64 " frames\n", source.format_name, (unsigned)source.channels, source.channels == 1U ? "" : "s", source.sample_rate, source.total_frames, MODEL_SAMPLE_RATE, resampler.output_frames); fprintf(stderr, "Model: %s\n", model_path); if (!model_init(&model, model_path, input, &error)) { exit_code = EXIT_MODEL; goto cleanup; } if (!output_set_create(&outputs, output_directory, command.identifier, resampler.output_frames, &error)) { exit_code = EXIT_OUTPUT; goto cleanup; } if (!separate_audio(&model, &resampler, &outputs, input, accumulator, weight, window, &exit_code, &error)) { goto cleanup; } if (!output_set_commit(&outputs, &error)) { exit_code = EXIT_OUTPUT; goto cleanup; } fprintf(stderr, "Wrote six stems to %s\n", output_directory); exit_code = EXIT_OK; cleanup: if (exit_code != EXIT_OK && error.message[0] != '\0') { fprintf(stderr, "error: %s\n", error.message); } output_set_cleanup(&outputs); model_destroy(&model); free(window); free(weight); free(accumulator); free(input); free(output_directory); free(model_path); resampler_destroy(&resampler); audio_source_close(&source); return exit_code; }