baked_graph_processor.hpp

Go to the documentation of this file.

#pragma once
 
// Lowering a fixed SignalGraph into a single shippable Processor.
//
// `bake()` freezes a prepared, fully-lowerable SignalGraph into one
// BakedGraphProcessor: a self-contained pulp::format::Processor that drives a
// frozen plan through the SAME canonical GraphRuntimeExecutor::process_routed()
// the live graph uses, so its output is bit-identical to the live graph's walk
// for the lowerable subset. There is no codegen and no second routing backend —
// the baked Processor only CALLS process_routed, never defines it.
//
// Lowerable: AudioInput, AudioOutput, the built-in Gain utility, and a Custom
// node whose registered type opts in with `lowerable = true`, matches the node's
// port shape, and is transport-independent (the baked process() drops the host
// transport, so a transport-sensitive node would diverge).
// A graph is REFUSED loudly (null processor + a reason) rather than silently
// mis-baked when it is not prepared, not executor-eligible, or carries a hosted
// Plugin node (opaque external state, not self-contained).
// A plan can be kept in memory (bake) or serialized to a signed .pulpbake
// envelope and read back (bake_to_plan -> write_baked_signed -> load_baked); see
// baked_codec.hpp for the on-disk format and its trust gate.
//
// bake() captures the graph's TOPOLOGY and Gain VALUES, not hot runtime state: the
// baked Processor builds fresh feedback/delay/scratch state in prepare() and starts
// it from zero. If the source graph has already processed blocks (non-zero feedback
// history), that history is not cloned — the baked Processor begins a fresh stream.
 
#include <pulp/format/graph_runtime_executor.hpp>
#include <pulp/format/processor.hpp>
#include <pulp/host/baked_codec.hpp>
#include <pulp/host/sample_region_parameters.hpp>
#include <pulp/host/signal_graph.hpp>
#include <pulp/host/signal_graph_executor_routing.hpp>
 
#include <atomic>
#include <cstddef>
#include <cstdint>
#include <functional>
#include <memory>
#include <span>
#include <string>
#include <type_traits>
#include <unordered_map>
#include <utility>
#include <vector>
 
namespace pulp::host {
 
namespace detail {
// Per-node injection substrate; full definitions live in the .cpp so this
// widely-included header stays free of param_cursor/store.
struct BakedParamMailbox;
struct BakedParamNodeState;
struct BakedCustomNodeRuntime;
struct BakedSampleRegionRuntime;
}  // namespace detail
 
// Bake-layer parameter-injection binding captured at bake() for one param-
// declaring custom node: the bound param-aware process callback (instance
// keepalive captured) plus the node's declared params. Handed to the
// BakedGraphProcessor so it can size the node's mailbox + build its cursor.
struct BakedCustomParamBinding {
    CustomNodeParamProcessFn process;
    std::vector<CustomNodeBakedParam> params;
};
 
// Why a graph could not be lowered into a self-contained BakedGraphProcessor.
enum class LowerRejectReason {
    None,
    NotPrepared,                  // graph.prepare() has not published a snapshot
    NotExecutorEligible,          // outside the routed executor's bit-exact subset
    HostedPluginNotSelfContained, // a Plugin node carries opaque external state
    CustomNotYetLowerable,        // a Custom node is unresolved, has no lowering yet,
                                  // or its shape does not match its registered type
    CustomNotLowerable,           // a Custom type is not opted into baking (lowerable=false)
    CustomTransportNotLowerable,  // a transport-sensitive Custom node (baked process
                                  // drops the transport, so it would diverge)
    NonAudioLaneNotLowerable,     // a MIDI node, or a MIDI/automation/sidechain edge
    CodecRejected,                // load_baked: the .pulpbake bytes failed the signed
                                  // envelope (bad magic/length/manifest/trust/signature/
                                  // hash) or the bounded plan parse
    StatefulCustomNotYetLoadable, // load_baked: matching Custom instance could
                                  // not be created or rejected authenticated state.
    ParameterContractMismatch,    // promoted parameter identities or metadata are invalid
    SpectralPersistenceUnsupported,
};
 
// Result of bake(): on success `processor` is non-null and `accepted` is true;
// on refusal `processor` is null, `accepted` is false, and `reason` /
// `offending_node` / `message` explain the refusal loudly.
struct LowerResult {
    std::unique_ptr<pulp::format::Processor> processor;
    bool accepted = false;
    LowerRejectReason reason = LowerRejectReason::None;
    NodeId offending_node = 0;
    std::string message;
};
 
// Proof that a graph topology is lowerable into a self-contained baked Processor.
// The single gate both bake() and (later) the on-disk load path consult, so the
// trusted and untrusted paths can never diverge on what is bakeable. accepted==true
// means every node kind is bakeable (audio I/O + Gain; Plugin/Custom/MIDI refused
// with a specific reason), every connection is plain audio, and the topology is
// executor-eligible. Pure — no allocation, no graph state. bake() calls it after
// its is_prepared() precondition.
struct LowerabilityProof {
    bool accepted = false;
    NodeId offending_node = 0;
    LowerRejectReason reason = LowerRejectReason::None;
    std::string message;
};
 
// `resolve_custom`, when set, resolves a Custom node's registered type (by
// custom_type_id + custom_type_version) so the gate can accept a lowerable,
// shape-matched, non-transport-sensitive Custom node. When empty (the default) any
// Custom node is refused as CustomNotYetLowerable — the in-process bake path passes
// a resolver over the graph's registry; a caller with no registry omits it.
LowerabilityProof lowerability_of(
    std::span<const GraphNode> nodes,
    std::span<const Connection> connections,
    const std::function<const CustomNodeType*(std::string_view type_id, int version)>&
        resolve_custom = {});
 
// Per-Custom-node lifecycle hooks captured at bake() from the node's registered
// type + live instance. prepare() runs them (control thread) so a re-prepare is a
// real re-init boundary: the instance is re-prepared at the HOST's actual
// rate/block (not whatever the source graph — or load_baked's nominal
// 48k/512 — last used) and its DSP state is reset, honoring the "starts a
// fresh stream" contract above. Either fn may be empty (type registered no
// such hook); each closure holds the instance shared_ptr, so it is also a
// keepalive.
struct CustomNodeLifecycle {
    std::function<void(double sample_rate, int max_block)> prepare;
    std::function<void()> reset;
    // Present only for a processor reconstructed from a signed artifact carrying
    // initial Custom state. Runs after prepare/reset so those hooks cannot erase
    // the authenticated state. False leaves the processor unprepared.
    std::function<bool()> restore_state;
};
 
// Complete bake-time binding for one Custom node. Keeping process, lifecycle,
// parameter support, and latency under one NodeId-keyed record prevents the
// four parallel maps from drifting as Custom-node capabilities evolve.
struct BakedCustomNodeBinding {
    CustomNodeProcessFn process;
    CustomNodeLifecycle lifecycle;
    BakedCustomParamBinding params;
    std::function<int(double)> latency_samples;
};
 
// Outcome of a ParamInjector::inject() call. Distinguishes the two failure
// modes the old bool return conflated: a dead handle (never call inject() on an
// invalid() injector) versus a source queue that overflowed before it reached
// us (the retained prefix is still published — a real, usable partial success).
enum class InjectStatus {
    Ok,               // published in full
    PartialOverflow,  // queue path: the source queue overflowed; its retained
                      // prefix is published. Single path: the pending batch was
                      // full of other params' events; this event was dropped.
    InvalidHandle,    // this injector holds no live claim (valid() == false); nothing published
};
 
// Exclusive control-side handle for injecting parameters into ONE baked custom
// node. Obtain from BakedGraphProcessor::claim_param_injection(); move-only;
// releases the node's exclusive claim on destruction. inject() is RT-safe and
// allocation-free — it publishes into the node's single-writer mailbox, which
// the baked process() drains on its next block. Publication semantics:
// inject(queue) REPLACES the pending batch (latest-published-queue wins, like
// the live mailbox); inject(single) ACCUMULATES — the new event merges into
// the still-unconsumed pending batch, superseding only that param's pending
// entries, so N single injects to different params on one node between blocks
// ALL land (they do not collapse to the last one). WITHIN a batch events are
// sample-accurate and may ramp (ParamCursor over sample_offset +
// ramp_duration_sample_frames), and a ramp longer than one block continues to
// completion across subsequent blocks. Exactly one injector may hold a node at
// a time (two owners injecting into one node is the hazard this claim prevents).
class ParamInjector {
public:
    ParamInjector() noexcept = default;
    ~ParamInjector();
    ParamInjector(ParamInjector&& other) noexcept;
    ParamInjector& operator=(ParamInjector&& other) noexcept;
    ParamInjector(const ParamInjector&) = delete;
    ParamInjector& operator=(const ParamInjector&) = delete;
 
    // True iff this handle holds a live exclusive claim on a node's mailbox.
    bool valid() const noexcept { return mailbox_ != nullptr; }
 
    // Publish a whole queue of sample-accurate events (the general path).
    // Returns InvalidHandle iff !valid() (nothing published — this is a caller
    // error, inject() is meant to be called only on a valid() handle), or
    // PartialOverflow when `events` already reports source-side overflow (the
    // retained prefix is still published — a usable partial success), otherwise
    // Ok. RT-safe / allocation-free.
    InjectStatus inject(const state::ParameterEventQueue& events) noexcept;
    // Convenience: publish a single immediate (ramp_duration==0) or ramped
    // event. Accumulates: merges into the pending (not-yet-consumed) batch,
    // superseding only THIS param's pending entries — consecutive single
    // injects to different params between blocks all apply. Returns
    // PartialOverflow only in the pathological case where the pending batch is
    // already full of OTHER params' events (a full-capacity queue publish, not
    // yet consumed): the pending batch stays published unchanged and this
    // event is dropped. Otherwise Ok / InvalidHandle.
    InjectStatus inject(const state::ParameterEvent& event) noexcept;
 
private:
    friend class BakedGraphProcessor;
    explicit ParamInjector(std::shared_ptr<detail::BakedParamMailbox> mailbox) noexcept;
    void release() noexcept;
    std::shared_ptr<detail::BakedParamMailbox> mailbox_;
};
 
// A SignalGraph frozen into a shippable Processor. Owns the reconstructed plan
// (nodes + connections), its own heap-stable Gain atomics, and the canonical
// executor's serialized routing snapshot + scratch pool. process() bridges the
// host's main in/out buffers into a ProcessBlock and runs the frozen plan via
// GraphRuntimeExecutor::process_routed(). It must NOT define process_routed /
// process_parallel (single-backend invariant) — only call them.
//
// In-place hosts: process() supports input and output views over the SAME
// memory (Logic-style aliased buffers). process_routed zeroes the main output
// bus before gathering AudioInput, which would destroy an aliased input — so
// process() detects overlap and reads the input from a prepare()-sized scratch
// copy instead (audio thread does only the memcpy; no allocation).
class BakedGraphProcessor : public pulp::format::Processor {
public:
    BakedGraphProcessor(std::vector<GraphNode> nodes,
                        std::vector<Connection> connections,
                        int input_channels,
                        int output_channels,
                        std::string name,
                        std::string bundle_id,
                        // One complete binding per Custom node, consolidating
                        // what were four parallel NodeId-keyed maps (process
                        // callback, intrinsic latency, lifecycle hooks, and
                        // bake-layer param injection) into a single record that
                        // cannot fall out of step across them. Callback copies
                        // capture their instance keepalive, so the baked
                        // Processor owns all required runtime state and holds no
                        // reference back into the source graph.
                        std::unordered_map<NodeId, BakedCustomNodeBinding>
                            custom_nodes = {});
    // Validate and freeze promoted metadata before a Processor can be returned
    // to an adapter. Invalid contracts return ParameterContractMismatch with no
    // processor; this does not replace the graph's topology/lowering proof.
    static LowerResult
    create_with_sample_regions(std::vector<GraphNode> nodes, std::vector<Connection> connections,
                               int input_channels, int output_channels, std::string name,
                               std::string bundle_id,
                               std::unordered_map<NodeId, BakedCustomNodeBinding> custom_nodes,
                               std::vector<SampleRegionDefinition> sample_regions);
 
    // Own exact scalar descriptors independently of the source graph registry.
    static LowerResult
    create_with_sample_regions(std::vector<GraphNode> nodes, std::vector<Connection> connections,
                               int input_channels, int output_channels, std::string name,
                               std::string bundle_id,
                               std::unordered_map<NodeId, BakedCustomNodeBinding> custom_nodes,
                               std::vector<SampleRegionDefinition> sample_regions,
                               std::vector<SampleKernelDescriptor> sample_kernels);
 
    ~BakedGraphProcessor() override;
 
    pulp::format::PluginDescriptor descriptor() const override;
    int latency_samples() const override {
        return prepared_latency_samples_.load(std::memory_order_relaxed);
    }
    void define_parameters(pulp::state::StateStore& store) override;
    const SampleRegionParameterContract& sample_region_parameter_contract() const noexcept {
        return sample_region_parameter_contract_;
    }
    const SampleRegionParameterBinding* sample_region_parameter_binding() const noexcept {
        return sample_region_parameter_binding_.get();
    }
    void prepare(const pulp::format::PrepareContext& context) override;
    void process(pulp::audio::BufferView<float>& audio_output,
                 const pulp::audio::BufferView<const float>& audio_input,
                 pulp::midi::MidiBuffer& midi_in,
                 pulp::midi::MidiBuffer& midi_out,
                 const pulp::format::ProcessContext& context) override;
 
    // Claim exclusive parameter-injection rights for a baked custom node. Returns
    // a valid ParamInjector iff `node` is a param-declaring baked custom node AND
    // no injector currently holds it; otherwise returns an invalid handle. The
    // returned handle survives prepare()/re-prepare (the mailbox persists across
    // both). Control thread only.
    ParamInjector claim_param_injection(NodeId node) noexcept;
 
private:
  BakedGraphProcessor(std::vector<GraphNode> nodes, std::vector<Connection> connections,
                      int input_channels, int output_channels, std::string name,
                      std::string bundle_id,
                      std::unordered_map<NodeId, BakedCustomNodeBinding> custom_nodes,
                      SampleRegionParameterContract parameter_contract);
 
  // Build per-node injection state and install the draining wrapper into each
  // param-declaring Custom runtime. Called by prepare()
  // before the executor snapshot is built; off the audio thread.
  void prepare_param_injection();
 
  // The frozen plan, captured at bake() time and owned for the Processor's
  // lifetime. Gain values ride GraphNode::gain; prepare() seeds the atomics.
  std::vector<GraphNode> nodes_;
  std::vector<Connection> conns_;
 
  // Heap-stable per-Gain-node atomics (one unique_ptr each) so the routed Gain
  // binding's user_data pointer survives gains_ growth. Built in prepare().
  std::vector<std::unique_ptr<std::atomic<float>>> gains_;
  std::unordered_map<NodeId, std::size_t> gain_index_;
 
  // Canonical executor + the serialized fused plan it runs. The snapshot and
  // pool are sized once in prepare() (off the audio thread); process() only
  // calls process_routed, which is allocation-free for a fitting pool.
  pulp::format::GraphRuntimeExecutor executor_;
  pulp::format::GraphRuntimeSnapshot snapshot_;
  pulp::format::GraphRuntimeBufferPool pool_;
  // Empty for this slice's lowerable subset (no Plugin nodes), but the
  // snapshot builder requires the storage to exist.
  std::vector<PluginBindingContext> plugin_ctx_;
  PluginRoutingScratch plugin_scratch_;
  // One owned runtime record per Custom node, consolidating its process,
  // lifecycle, parameter injection, mailbox/state, and latency callback. Each
  // record holds its own instance keepalive. Empty for a custom-free graph;
  // prepare() binds custom_ctx_ from these records.
  std::unordered_map<NodeId, std::unique_ptr<detail::BakedCustomNodeRuntime>> custom_nodes_;
  std::vector<CustomBindingContext> custom_ctx_;
 
  // In-place-host guard scratch: when the host's input channels alias its
  // output channels, process() copies the input here BEFORE process_routed
  // zeroes the output bus. Sized in prepare() (input_channels_ × max block);
  // the audio thread only detects overlap and memcpys — no allocation.
  std::vector<float> input_alias_scratch_;
  std::vector<float*> input_alias_ptrs_;
 
  std::string name_;
  std::string bundle_id_;
  SampleRegionParameterContract sample_region_parameter_contract_;
  std::unique_ptr<SampleRegionParameterBinding> sample_region_parameter_binding_;
  std::unique_ptr<detail::BakedSampleRegionRuntime> sample_region_runtime_;
  int input_channels_ = 2;
  int output_channels_ = 2;
  int prepared_max_block_ = 0;
  // Resolved at prepare(), not at construction: a Custom node's intrinsic
  // latency is a function of the sample rate, which is not known until then.
  std::atomic<int> prepared_latency_samples_{0};
  bool prepared_ = false;
};
 
// Lower a prepared, fully-lowerable SignalGraph into a BakedGraphProcessor.
// Returns LowerResult{accepted=true, processor=...} on success, or a loud
// refusal (processor=nullptr) with a reason for any non-lowerable graph.
LowerResult bake(const SignalGraph& graph);
 
// Verify + load a signed .pulpbake artifact into a BakedGraphProcessor. `trust` is
// the set of accepted signer keys; `custom_types` are the host's registered custom
// node types (a Custom record's process code is re-resolved from these, never from
// the file). The bytes go through verify_and_extract_plan (signature-before-parse),
// then the plan is rebuilt into a SignalGraph and run through bake() — so bake()'s
// full lowerability re-proof applies to the reconstructed topology; the file's
// implicit claim is never trusted. Returns CodecRejected if the envelope/parse fails,
// or bake()'s refusal reason if the reconstructed graph is not lowerable. v1 supports
// stateful custom nodes when the matching registered type supplies create +
// load_state. The authenticated blob is restored after each host prepare/reset;
// malformed state fails closed before a processor is returned.
LowerResult load_baked(std::span<const std::uint8_t> bytes, const BakedTrust& trust,
                       const std::vector<CustomNodeType>& custom_types);
 
// Exact block/scalar registration pair used by the bake-v2 preflight. This is
// deliberately separate from CustomNodeType so v1 callers retain the original
// source surface and a v2 artifact can never resolve sample code by block type
// identity alone. Every supplied pair must be valid; an identity used only
// by residual Custom nodes resolves through its block registration internally.
struct BakedTypeRegistration {
    CustomNodeType block;
    SampleKernelDescriptor sample;
};
 
// Named, non-default wrapper keeps the paired overload distinct from legacy
// calls such as load_baked(bytes, trust, {}).
class BakedTypeRegistry {
  public:
    BakedTypeRegistry() = delete;
    static BakedTypeRegistry from(std::vector<BakedTypeRegistration> registrations) {
        return BakedTypeRegistry(std::move(registrations), ConstructionKey{});
    }
 
    const std::vector<BakedTypeRegistration>& registrations() const noexcept {
        return registrations_;
    }
 
  private:
    struct ConstructionKey {};
    BakedTypeRegistry(std::vector<BakedTypeRegistration> registrations, ConstructionKey)
        : registrations_(std::move(registrations)) {}
 
    std::vector<BakedTypeRegistration> registrations_;
};
 
// Verify a signed v2 artifact, reconstruct its authored topology and region
// metadata in an isolated PreparedTopologyEdit, and re-run exact region and
// ordinary graph proofs. Inspection returns the callback-free verified plan;
// it does not publish or execute a runtime snapshot.
struct BakedPlanLoadResult {
    std::optional<BakedPlan> plan;
    bool accepted = false;
    LowerRejectReason reason = LowerRejectReason::None;
    SampleRegionId offending_region = 0;
    NodeId offending_node = 0;
    std::string message;
};
 
BakedPlanLoadResult load_baked_plan(std::span<const std::uint8_t> bytes, const BakedTrust& trust,
                                    const std::vector<BakedTypeRegistration>& registrations);
 
namespace detail {
LowerResult load_baked_registered(std::span<const std::uint8_t> bytes, const BakedTrust& trust,
                                  const BakedTypeRegistry& registry);
}
 
// Paired registrations admit executable signed sample regions with private
// scalar state. Built-in kernels need no extra registrations. Call
// define_parameters(store) before prepare(), including for an empty manifest;
// the adapter store must outlive the processor. An empty brace argument still
// selects the legacy v1-only vector overload. Residual Custom callbacks own
// their block instances and authenticated state; a failed opaque state restore
// on reprepare retains the legacy fail-silent behavior, not region-bank rollback.
template <typename Registry>
    requires std::is_same_v<std::remove_cvref_t<Registry>, BakedTypeRegistry>
LowerResult load_baked(std::span<const std::uint8_t> bytes, const BakedTrust& trust,
                       Registry&& registry) {
    return detail::load_baked_registered(bytes, trust, registry);
}
 
// Result of bake_to_plan(): `plan` is set iff `accepted`; on refusal the reason /
// offending_node / message explain why (mirrors bake()'s LowerResult so the write
// front door can report a non-shippable graph as precisely as the in-memory path).
struct BakePlanResult {
    std::optional<BakedPlan> plan;
    bool accepted = false;
    LowerRejectReason reason = LowerRejectReason::None;
    NodeId offending_node = 0;
    std::string message;
};
 
// Extract a serializable BakedPlan from a prepared, lowerable graph — the write-path
// front door: bake_to_plan(graph).plan -> write_baked_signed(*plan, key) -> bytes.
// Refuses (accepted=false, plan=nullopt) with a reason if the graph is not prepared
// or not lowerable (same proof as bake()). Custom state is the authored blob last
// supplied through set_custom_node_state(), not a live save_state() snapshot: a
// prepared graph may be processing concurrently and save_state has no RT concurrency
// contract.
BakePlanResult bake_to_plan(const SignalGraph& graph);
 
} // namespace pulp::host