Candle & Memory-Mapped Safetensors

Zero-Copy Tensor Loading in Rust

Hugging Face's Candle is a minimalist, performant ML framework for Rust. Using memory-mapped (mmap) Safetensors files, you can map multi-gigabyte model weights directly from NVMe storage into the process address space without copying them through user-space RAM buffers.

use candle_core::{DType, Device, Tensor};
use candle_core::safetensors::MmapedSafetensors;
use std::path::Path;

pub struct RustInferenceHarness {
    device: Device,
    tensors: MmapedSafetensors,
}

impl RustInferenceHarness {
    pub fn new>(weight_path: P, use_cuda: bool) -> anyhow::Result {
        let device = if use_cuda && candle_core::cuda::is_available() {
            Device::new_cuda(0)?
        } else {
            Device::Cpu
        };

        // Zero-copy memory map of the model weights file
        let tensors = unsafe { MmapedSafetensors::new(weight_path)? };
        
        Ok(Self { device, tensors })
    }

    pub fn load_tensor(&self, name: &str) -> candle_core::Result {
        self.tensors.load(name, &self.device)
    }
}