VGPU-IO Integration Guide

vGPU-IO Integration Guide

Historical document — superseded. This earlier integration design predates the current stack and is retained for reference only. Canonical documentation: getting started ../docs/02-getting-started/, host module ../docs/03-architecture/06-host-module-and-hybrid-model.md, developer standards ../docs/07-developers/.

Overview

This guide covers implementing a Linux kernel driver for vGPU-IO using the GSP 570.144W firmware RPC interface.

Architecture

┌─────────────────────────────────────────────────────────────┐
│                    vGPU-IO Kernel Driver                     │
├─────────────────────────────────────────────────────────────┤
│  gsp_core.c      │  gsp_rpc.c    │  gsp_vf.c               │
│  - Device probe  │  - RPC client │  - VF lifecycle         │
│  - BAR mapping   │  - Doorbell   │  - Migration            │
│  - IRQ/MSI-X     │  - Completion │  - WPR/BAR integration  │
├─────────────────────────────────────────────────────────────┤
│                    GSP Firmware (570.144W)                   │
│  Primary Dispatcher (0x057A1AC0)  │  Secondary (0x04ED39DC) │
│  BAR Handler (0x04CB8188)         │  WPR Handlers           │
│  Migration (0x04179ED8/0x04179FBC)│  SCHED/ALLOC/PIN        │
└─────────────────────────────────────────────────────────────┘

Module Structure

drivers/gpu/nvidia/gsp/
├── gsp.h              # Core definitions
├── gsp_rpc.h          # RPC message formats
├── gsp_vf.h           # VF definitions
├── gsp_bar.h          # BAR emulation
├── gsp_wpr.h          # WPR management
├── gsp_migration.h    # Migration protocol
├── gsp_core.c         # Probe/remove, BAR/IRQ
├── gsp_rpc.c          # RPC client implementation
├── gsp_vf.c           # VF create/destroy
├── gsp_bar.c          # BAR trap handler
├── gsp_wpr.c          # WPR protect/unprotect
├── gsp_migration.c    # Live migration
└── gsp_debugfs.c      # Debug interface

Core Data Structures

GSP Device

struct gsp_device {
    struct pci_dev *pdev;
    void __iomem *bar0;           // PCIe BAR0 mapping
    void __iomem *bar1;           // PCIe BAR1 (GSP FW)
    struct gsp_firmware *fw;      // Firmware image
    struct gsp_rpc_client *rpc;   // RPC client
    struct mutex vf_lock;
    struct idr vf_idr;            // VF ID allocator
    struct list_head vf_list;
    struct gsp_debugfs *debugfs;
    spinlock_t doorbell_lock;
    uint32_t doorbell_seq;
};

VF (Virtual Function)

struct gsp_vf {
    struct gsp_device *gsp;
    uint32_t id;                  // VF ID (1-7)
    struct kref ref;
    struct mutex lock;
    mig_state_t mig_state;
    struct gsp_migration *migration;
    uint64_t bar1_offset;         // BAR1 aperture offset
    uint64_t bar1_size;           // BAR1 aperture size
    uint64_t pt_base;             // Page table base
    uint64_t pt_size;             // Page table size
    uint64_t cmd_base;            // Command buffer base
    uint64_t cmd_size;            // Command buffer size
    struct gsp_wpr_region *wpr_regions;
    struct list_head wpr_list;
    struct gsp_bar_state bar_state;
    atomic_t refcount;
    bool active;
};

Initialization Sequence

1. PCI Probe

static int gsp_probe(struct pci_dev *pdev, const struct pci_device_id *id) {
    struct gsp_device *gsp;
    int ret;
    
    gsp = devm_kzalloc(&pdev->dev, sizeof(*gsp), GFP_KERNEL);
    gsp->pdev = pdev;
    pci_set_drvdata(pdev, gsp);
    
    // Enable PCI device
    ret = pcim_enable_device(pdev);
    if (ret) return ret;
    
    // Map BARs
    gsp->bar0 = pcim_iomap(pdev, 0, 0);
    gsp->bar1 = pcim_iomap(pdev, 1, 0);
    if (!gsp->bar0 || !gsp->bar1) return -ENOMEM;
    
    // Load firmware
    ret = gsp_firmware_load(gsp, "gsp-570.144.fwimage.bin");
    if (ret) return ret;
    
    // Initialize RPC client
    gsp->rpc = gsp_rpc_client_init(gsp);
    if (IS_ERR(gsp->rpc)) return PTR_ERR(gsp->rpc);
    
    // Setup IRQ/MSI-X
    ret = gsp_setup_irq(gsp);
    if (ret) return ret;
    
    // Initialize debugfs
    gsp_debugfs_init(gsp);
    
    return 0;
}

2. Firmware Loading

int gsp_firmware_load(struct gsp_device *gsp, const char *name) {
    const struct firmware *fw;
    int ret;
    
    ret = request_firmware(&fw, name, &gsp->pdev->dev);
    if (ret) return ret;
    
    // Verify MD5: 13ce12a392a31eeb79a2e6ebd3a4a93a
    if (!gsp_firmware_verify(fw->data, fw->size)) {
        release_firmware(fw);
        return -EINVAL;
    }
    
    // Load to GSP memory (via BAR1)
    ret = gsp_firmware_upload(gsp, fw->data, fw->size);
    release_firmware(fw);
    
    return ret;
}

3. GSP Boot

int gsp_boot(struct gsp_device *gsp) {
    // 1. Reset GSP
    writel(1, gsp->bar0 + GSP_RESET_REG);
    msleep(10);
    writel(0, gsp->bar0 + GSP_RESET_REG);
    
    // 2. Wait for entry point (0x04009000) to signal ready
    ret = gsp_wait_for_ready(gsp, 5000);  // 5s timeout
    if (ret) return ret;
    
    // 3. Initialize RPC dispatchers
    ret = gsp_rpc_init_dispatchers(gsp);
    if (ret) return ret;
    
    return 0;
}

RPC Client Implementation

Doorbell Mechanism

#define GSP_DOORBELL_OFFSET   0x1000
#define GSP_MSG_RING_SIZE     256
#define GSP_MSG_MAX_SIZE      256

struct gsp_rpc_client {
    struct gsp_device *gsp;
    struct mutex lock;
    uint32_t seq;
    wait_queue_head_t wq;
    struct gsp_msg_ring *ring;
    struct completion *pending[GSP_MAX_PENDING];
};

static int gsp_rpc_send_sync(struct gsp_rpc_client *rpc,
                             uint32_t rpc_type,
                             void *payload, size_t payload_size,
                             void *response, size_t *response_size) {
    struct gsp_msg msg = {
        .hdr = {
            .magic = 0x47535052,
            .version = 1,
            .rpc_type = rpc_type,
            .sequence = atomic_inc_return(&rpc->seq),
            .timestamp_ns = ktime_get_ns(),
            .payload_size = payload_size,
            .flags = MSG_SYNC,
        }
    };
    
    struct completion done;
    init_completion(&done);
    
    mutex_lock(&rpc->lock);
    rpc->pending[msg.hdr.sequence % GSP_MAX_PENDING] = &done;
    
    // Copy message to ring buffer
    memcpy(msg.hdr.payload, payload, payload_size);
    gsp_ring_write(rpc->ring, &msg, GSP_MSG_MAX_SIZE);
    
    // Ring doorbell
    writel(1, rpc->gsp->bar0 + GSP_DOORBELL_OFFSET);
    
    mutex_unlock(&rpc->lock);
    
    // Wait for completion
    if (!wait_for_completion_timeout(&done, msecs_to_jiffies(5000)))
        return -ETIMEDOUT;
    
    // Copy response
    if (response && response_size) {
        *response_size = min(*response_size, done.payload_size);
        memcpy(response, done.payload, *response_size);
    }
    
    return done.status;
}

Completion Handler (ISR)

static irqreturn_t gsp_isr(int irq, void *data) {
    struct gsp_device *gsp = data;
    struct gsp_msg msg;
    
    while (gsp_ring_read(gsp->rpc->ring, &msg)) {
        if (msg.hdr.flags & MSG_SYNC) {
            struct completion *done = 
                gsp->rpc->pending[msg.hdr.sequence % GSP_MAX_PENDING];
            if (done) {
                done->status = msg.payload.status;
                done->payload_size = msg.hdr.payload_size;
                memcpy(done->payload, msg.payload.data, msg.hdr.payload_size);
                complete(done);
            }
        } else {
            // Async notification
            gsp_handle_notification(gsp, &msg);
        }
    }
    
    return IRQ_HANDLED;
}

VF Lifecycle

Create VF

int gsp_vf_create(struct gsp_device *gsp, uint32_t vf_id,
                  const struct gsp_vf_config *config) {
    struct gsp_vf *vf;
    int ret;
    
    mutex_lock(&gsp->vf_lock);
    
    if (idr_alloc(&gsp->vf_idr, NULL, vf_id, vf_id + 1, GFP_KERNEL) < 0) {
        ret = -EBUSY;
        goto unlock;
    }
    
    vf = kzalloc(sizeof(*vf), GFP_KERNEL);
    vf->gsp = gsp;
    vf->id = vf_id;
    kref_init(&vf->ref);
    mutex_init(&vf->lock);
    INIT_LIST_HEAD(&vf->wpr_list);
    
    // Configure VF resources
    vf->bar1_offset = config->bar1_offset;
    vf->bar1_size = config->bar1_size;
    vf->pt_base = config->pt_base;
    vf->pt_size = config->pt_size;
    vf->cmd_base = config->cmd_base;
    vf->cmd_size = config->cmd_size;
    
    // Initialize WPR for VF
    ret = gsp_vf_wpr_init(vf);
    if (ret) goto err_wpr;
    
    // Initialize BAR for VF
    ret = gsp_vf_bar_init(vf);
    if (ret) goto err_bar;
    
    list_add(&vf->list, &gsp->vf_list);
    vf->active = true;
    
    mutex_unlock(&gsp->vf_lock);
    return 0;
    
err_bar:
    gsp_vf_wpr_fini(vf);
err_wpr:
    idr_remove(&gsp->vf_idr, vf_id);
    kfree(vf);
unlock:
    mutex_unlock(&gsp->vf_lock);
    return ret;
}

Destroy VF

void gsp_vf_destroy(struct gsp_vf *vf) {
    struct gsp_device *gsp = vf->gsp;
    uint32_t id = vf->id;
    
    mutex_lock(&gsp->vf_lock);
    vf->active = false;
    list_del(&vf->list);
    mutex_unlock(&gsp->vf_lock);
    
    gsp_vf_bar_fini(vf);
    gsp_vf_wpr_fini(vf);
    
    if (vf->migration)
        gsp_migration_cancel(vf->migration);
    
    kref_put(&vf->ref, gsp_vf_release);
}

static void gsp_vf_release(struct kref *ref) {
    struct gsp_vf *vf = container_of(ref, struct gsp_vf, ref);
    kfree(vf);
}

Memory Management (ALLOC/PIN)

Allocate VF Memory

int gsp_vf_mem_alloc(struct gsp_vf *vf, 
                     uint64_t size, uint32_t flags,
                     uint64_t *handle, uint64_t *va) {
    struct alloc_payload payload = {
        .size = size,
        .type = MEM_TYPE_VRAM,
        .alignment = 0x1000,
        .va_hint = 0,
        .flags = flags,
        .pid = current->pid,
        .vf_id = vf->id,
    };
    
    struct alloc_response resp;
    size_t resp_size = sizeof(resp);
    
    return gsp_rpc_send_sync(vf->gsp->rpc, RPC_ALLOC,
                             &payload, sizeof(payload),
                             &resp, &resp_size);
}

Pin Memory for DMA

int gsp_vf_mem_pin(struct gsp_vf *vf, uint64_t handle,
                   uint64_t offset, uint64_t size,
                   uint64_t *dma_addr) {
    struct pin_payload payload = {
        .handle = handle,
        .offset = offset,
        .size = size,
        .vf_id = vf->id,
        .flags = PIN_WRITE | PIN_READ,
    };
    
    struct pin_response resp;
    size_t resp_size = sizeof(resp);
    
    return gsp_rpc_send_sync(vf->gsp->rpc, RPC_PIN,
                             &payload, sizeof(payload),
                             &resp, &resp_size);
}

Debugging & Testing

Debugfs Interface

# List VFs
cat /sys/kernel/debug/gsp/vfs

# VF details
cat /sys/kernel/debug/gsp/vf/1/status

# RPC statistics
cat /sys/kernel/debug/gsp/rpc/stats

# BAR state
cat /sys/kernel/debug/gsp/vf/1/bar

# WPR regions
cat /sys/kernel/debug/gsp/vf/1/wpr

# Migration status
cat /sys/kernel/debug/gsp/vf/1/migration

Trace Events

# Enable tracing
echo 1 > /sys/kernel/debug/tracing/events/gsp/enable

# Trace migration
trace-cmd record -e gsp_mig_phase -e gsp_mig_transfer

# Trace WPR
trace-cmd record -e gsp_wpr_event

Unit Tests

// kselftest for gsp-rpc
static void test_rpc_alloc(void) {
    struct gsp_vf *vf = create_test_vf();
    uint64_t handle, va;
    
    ASSERT_EQ(gsp_vf_mem_alloc(vf, 4096, 0, &handle, &va), 0);
    ASSERT_NE(handle, 0);
    ASSERT_NE(va, 0);
    
    gsp_vf_mem_free(vf, handle);
    destroy_test_vf(vf);
}

Error Handling Checklist

Scenario	Detection	Recovery
GSP firmware crash	Watchdog timeout	Reset GSP, reload FW
RPC timeout	Doorbell no response	Retry, then VF reset
BAR trap storm	Rate limit exceeded	Throttle VF
WPR violation	Callback storm	Quarantine VF
Migration fail	Commit verify fail	Rollback to source

Performance Tuning

RPC Batching

// Batch multiple ALLOCs in single RPC
struct batch_alloc_payload {
    uint32_t count;
    struct alloc_payload allocs[MAX_BATCH];
};

Async Completion

// Use async RPC for non-critical paths
gsp_rpc_send_async(rpc, RPC_NOTIFY, payload, size, callback, ctx);

Memory Pool

// Pre-allocate bounce buffers
struct gsp_bounce_pool {
    struct page **pages;
    int count;
    spinlock_t lock;
};

---

Reference

• GSP Firmware: gsp-570.144.fwimage.bin
• Base Address: 0x04000000
• Primary Dispatcher: 0x057A1AC0
• Secondary Dispatcher: 0x04ED39DC