gpu_convolver.hpp

Go to the documentation of this file.

#pragma once
 
#include <algorithm>
#include <cmath>
#include <cstdint>
#include <memory>
#include <string>
#include <vector>
 
#include <pulp/gpu_audio/gpu_audio_node.hpp>
#include <pulp/gpu_audio/gpu_audio_status.hpp>
#include <pulp/signal/convolver.hpp>
#include <pulp/signal/fixed_ir_convolver.hpp>
 
#include <pulp/platform/gpu_fwd.hpp>
 
namespace pulp::gpu_audio {

struct GpuConvolverTraceConfig {
    bool enabled = false;
    bool capture_admissions = false;
    bool capture_callback_timing = false;
    std::uint32_t success_stride = 1;
};

struct GpuConvolverStagedBreaker {
    std::uint32_t trip_after_misses = 8;
    std::uint32_t probe_interval_blocks = 32;
    std::uint32_t recover_after_hits = 4;
};
 
class GpuConvolver;
 
namespace detail {
struct RealtimeGpuHealth;
struct SharedIoCompletionNotifier;
struct GpuConvolverTestAccess;
struct SharedIoProviderIdentity;
struct GpuConvolverTrialConfig;
struct SharedIoTraceRecord;
struct SharedIoTraceAdmission;
class StagedAsyncTrialState;
bool configure_gpu_convolver_trial(GpuConvolver&, const GpuConvolverTrialConfig&) noexcept;
bool drain_gpu_convolver_trial_records(GpuConvolver&, std::vector<SharedIoTraceRecord>&) noexcept;
bool drain_gpu_convolver_trial_admissions(GpuConvolver&,
                                          std::vector<SharedIoTraceAdmission>&) noexcept;
std::uint64_t gpu_convolver_trial_engine_id(const GpuConvolver&) noexcept;
} // namespace detail

class GpuConvolver : public GpuAudioNode {
  public:
    enum class ProviderPolicy : std::uint8_t {
        Auto,
        StagedOnly,
        SharedRequired,
        StagedBlocking,
    };

    static constexpr uint32_t kLatencyBlocks = 2;
    static constexpr uint32_t kMaxLatencyBlocks = 64;
    // Physical shared-I/O provider capacity, independent of logical lead.
    // Diagnostic receipts use this value instead of a stale literal.
    static constexpr uint32_t kSharedIoSlots = 16;
 
    GpuConvolver(uint32_t channels, uint32_t block_size, uint32_t sample_rate,
                 std::vector<float> impulse_response);
    GpuConvolver(uint32_t channels, uint32_t block_size, uint32_t sample_rate,
                 std::vector<float> impulse_response, uint32_t latency_blocks);
    ~GpuConvolver() override;

    bool set_provider_policy(ProviderPolicy policy) noexcept;
    ProviderPolicy provider_policy() const noexcept {
        return provider_policy_;
    }

    bool configure_trace(const GpuConvolverTraceConfig& config) noexcept;

    bool configure_staged_breaker(const GpuConvolverStagedBreaker& config) noexcept;
    GpuConvolverStagedBreaker staged_breaker() const noexcept;
 
    GpuAudioNodeDescriptor descriptor() const override;
    bool prepare() override;
 
    bool cpu_fallback_prepared() const noexcept override {
        return prepared_;
    }
    void process_block(const audio::BufferView<const float>& input,
                       audio::BufferView<float>& output, uint32_t n) override;

    void prime_fallback(const audio::BufferView<const float>& input, uint32_t n) noexcept override;

    void process_cpu_fallback(const audio::BufferView<const float>& /*input*/,
                              audio::BufferView<float>& output, uint32_t n) noexcept override {
        const uint32_t out_ch = static_cast<uint32_t>(output.num_channels());
        for (uint32_t ch = 0; ch < out_ch; ++ch)
            std::fill_n(output.channel_ptr(ch), n, 0.0f);
        if (!prepared_ || n != block_)
            return;
        const uint32_t ch_count = std::min<uint32_t>(channels_, out_ch);
        for (uint32_t ch = 0; ch < ch_count; ++ch) {
            const float* due = fb_out_.data() + static_cast<std::size_t>(ch) * block_;
            std::copy_n(due, n, output.channel_ptr(ch));
        }
    }

    bool gpu_available() const;
    std::string backend() const;
    uint32_t fft_size() const {
        return fft_size_;
    }

    GpuAudioStatus status_snapshot() const noexcept override;
    bool bind_realtime_path(detail::RealtimeGpuNodeBinding& binding) noexcept override;
    GpuAudioRecoveryReason recovery_reason() const noexcept;
    bool fenced() const noexcept;
    std::uint64_t sanitized_input_samples() const noexcept;
 
  protected:
    void init_fallback() {
        fallback_.clear();
        worker_fallback_.clear();
        fallback_.resize(channels_);
        worker_fallback_.resize(channels_);
        for (uint32_t ch = 0; ch < channels_; ++ch) {
            fallback_[ch].load_ir(ir_.data(), ir_.size(), block_);
            worker_fallback_[ch].load_ir(ir_.data(), ir_.size(), block_);
        }
        fallback_delay_blocks_ = latency_blocks_;
        fb_delay_idx_ = 0;
        fb_delay_.assign(static_cast<std::size_t>(fallback_delay_blocks_) * block_ * channels_,
                         0.0f);
        fb_out_.assign(static_cast<std::size_t>(block_) * channels_, 0.0f);
    }

    void render_worker_fallback(const audio::BufferView<const float>& input,
                                audio::BufferView<float>& output, uint32_t n) noexcept {
        const uint32_t out_ch = static_cast<uint32_t>(output.num_channels());
        for (uint32_t ch = 0; ch < out_ch; ++ch)
            std::fill_n(output.channel_ptr(ch), n, 0.0f);
        if (!prepared_)
            return;
        const uint32_t ch_count = std::min<uint32_t>(channels_, out_ch);
        for (uint32_t ch = 0; ch < ch_count && ch < worker_fallback_.size(); ++ch) {
            if (ch >= input.num_channels())
                continue;
            const float* x = input.channel_ptr(ch);
            bool finite = true;
            for (uint32_t i = 0; i < n; ++i) {
                if (!std::isfinite(x[i])) {
                    finite = false;
                    break;
                }
            }
            if (!finite) {
                worker_fallback_[ch].reset(); // output stays the pre-cleared silence
                continue;
            }
            worker_fallback_[ch].process(x, output.channel_ptr(ch), static_cast<std::size_t>(n));
        }
    }
 
    static std::uint8_t process_realtime_shared_io(void* self,
                                                   const audio::BufferView<const float>& input,
                                                   audio::BufferView<float>& output,
                                                   std::uint32_t n, std::uint64_t sequence,
                                                   bool input_valid,
                                                   std::uint64_t callback_start_ns) noexcept;
    static std::uint32_t service_realtime_shared_io(void* self, std::uint64_t now_ns) noexcept;
    static bool fence_realtime_shared_io(void* self) noexcept;
    static void complete_realtime_shared_io(void*, std::uint64_t, std::uint8_t,
                                            std::uint64_t callback_end_ns,
                                            std::uint64_t result_visible_ns) noexcept;
    static std::uint64_t next_realtime_shared_io_sequence(void*) noexcept;
    // Serialized non-RT (transport worker or quiescent host). Also republishes
    // the sample for status_snapshot() readers on other threads.
    static bool realtime_shared_io_health(const void* self, detail::RealtimeGpuHealth&) noexcept;
    // Host only, while no service step runs: route the shared session's
    // provider completion events to the transport worker.
    static bool set_realtime_shared_io_notifier(void* self,
                                                detail::SharedIoCompletionNotifier) noexcept;
    // Host-thread safe: the staged GpuCompute loss query is atomic.
    GpuAudioRecoveryReason staged_recovery_reason() const noexcept;
    bool has_realtime_shared_io() const noexcept;
    detail::SharedIoProviderIdentity provider_identity_for_diagnostics() const noexcept;
 
    struct StagedReadbackState;
    // One-block-lead fallback: submit the packed block and wait, bounded, for
    // its completion into `dest`. True only for a Success readback.
    bool wait_staged_readback(StagedReadbackState& staged, float* dest);
    // Pipelined path: submit the packed block into slot `slot_index` without
    // waiting; it is judged when due, pipeline_blocks calls later.
    void submit_staged_readback(StagedReadbackState& staged, std::size_t slot_index);
    // Choose the staged pipeline depth from the prepared device and policy and
    // size its slots. Called at the end of prepare(); a test that swaps in a
    // scripted device calls it again before the node is attached.
    void configure_staged_pipeline();
    // Pipeline depth the current hosting, device and policy call for.
    std::uint32_t staged_pipeline_depth() const noexcept;
    void latch_staged_recovery(GpuAudioRecoveryReason reason) noexcept;
    void update_staged_breaker(StagedReadbackState& staged, bool on_time) noexcept;
    // Blocks for which the staged worker ran its CPU convolution since
    // prepare() (a missed block or the tail after one). Diagnostic.
    std::uint64_t staged_cpu_blocks() const noexcept;
    // Blocks whose GPU result was used, and breaker trips, since prepare().
    // Diagnostic; worker-thread values, read when the worker is quiescent.
    std::uint64_t staged_gpu_blocks() const noexcept;
    std::uint64_t staged_breaker_trips() const noexcept;
    // First cause latched by the staged worker since prepare(): ProviderLost
    // (device loss; replaces any other cause), SustainedDeadlineMisses (the
    // breaker tripped on a run of late results), ProviderFailure (failed map or
    // non-finite readback), else None. A single late block latches nothing.
    // Any thread; the public status accessor reads it.
    GpuAudioRecoveryReason staged_latched_reason() const noexcept;
    // The transport's worker-pipelining opt-in, bound through
    // bind_realtime_path(); `self` is the GpuConvolver.
    static bool set_staged_pipeline_hosting(void* self, bool hosted) noexcept;
    friend bool
    detail::configure_gpu_convolver_trial(GpuConvolver&,
                                          const detail::GpuConvolverTrialConfig&) noexcept;
    friend bool
    detail::drain_gpu_convolver_trial_records(GpuConvolver&,
                                              std::vector<detail::SharedIoTraceRecord>&) noexcept;
    friend bool detail::drain_gpu_convolver_trial_admissions(
        GpuConvolver&, std::vector<detail::SharedIoTraceAdmission>&) noexcept;
    friend std::uint64_t detail::gpu_convolver_trial_engine_id(const GpuConvolver&) noexcept;
    friend struct detail::GpuConvolverTestAccess;
 
    struct SharedIoState;
 
    uint32_t channels_;
    uint32_t block_;
    uint32_t sample_rate_;
    // Fixed before prepare(); this value is shared by descriptor PDC, the
    // shared-I/O bridge lead, and the CPU fallback delay.
    uint32_t latency_blocks_ = kLatencyBlocks;
    ProviderPolicy provider_policy_ = ProviderPolicy::Auto;
    std::vector<float> ir_;
    uint32_t fft_size_ = 0;
    std::uint8_t trial_requested_path_ = 0; // detail::SharedIoRequest::Auto
    std::uint64_t trial_generation_ = 1;
    bool trial_configured_ = false;
    bool trial_enable_trace_ = false;
    bool trial_capture_admissions_ = false;
    bool trial_capture_callback_timing_ = false;
    bool trial_staged_sync_reference_ = false;
    std::uint32_t trial_success_stride_ = 1;
    std::uint8_t trial_completion_policy_ = 0; // Dawn completion policy enum
    // Low 32 bits carry the private completion wait; high 32 bits carry the
    // private trace-retention bound. Keeping both in this existing field
    // preserves the exported class layout for SDK consumers.
    std::uint64_t trial_completion_wait_ns_ = 0;
 
    std::unique_ptr<render::GpuCompute> gpu_;
    std::unique_ptr<SharedIoState> shared_io_;
    std::unique_ptr<detail::StagedAsyncTrialState> staged_trial_;
    // Quiescent-only records for the explicit P4 staged_sync reference.
    std::vector<detail::SharedIoTraceRecord> staged_sync_records_;
    std::uint64_t staged_sequence_ = 0;
    std::vector<float> ir_spec_;            // 2*fft_size interleaved IR spectrum
    std::vector<std::vector<float>> carry_; // per-channel OLA accumulator (fft_size)
 
    // Per-block host scratch (allocated in prepare()). The FFT/mul/inverse
    // intermediates stay GPU-resident inside GpuCompute::convolve().
    std::vector<float> in_pad_; // 2*fft_size interleaved complex input
    std::vector<float> time_;   // 2*fft_size inverse result (one readback)
 
    // Continuously-fed zero-latency CPU fallbacks.
    std::vector<signal::FixedIrConvolver> fallback_;        // RT miss fallback
    std::vector<signal::FixedIrConvolver> worker_fallback_; // no-GPU worker path
 
    // Latency-alignment delay for the RT miss fallback. `fb_delay_` is a
    // `fallback_delay_blocks_`-deep circular buffer of block-sized, per-channel
    // slots; `fb_out_` holds this block's due (input(t-L)-convolved) substitute.
    uint32_t fallback_delay_blocks_ = 0;
    uint32_t fb_delay_idx_ = 0;
    std::vector<float> fb_delay_; // [delay_blocks][channels][block]
    std::vector<float> fb_out_;   // [channels][block]
 
    bool prepared_ = false;
 
    // Staged worker state: breaker settings, the pipelined readback slots, the
    // CPU convolution's owed tail and the latched recovery cause. Allocated by
    // the constructor so settings made before prepare() have a home.
    std::unique_ptr<StagedReadbackState> staged_readback_;
};
 
} // namespace pulp::gpu_audio