krfb/framebuffers/pipewire/pw_framebuffer.cpp

/* This file is part of the KDE project
   Copyright (C) 2018 Oleg Chernovskiy <kanedias@xaker.ru>

   This program is free software; you can redistribute it and/or
   modify it under the terms of the GNU General Public
   License as published by the Free Software Foundation; either
   version 3 of the License, or (at your option) any later version.
*/

// system
#include <sys/mman.h>
#include <cstring>
// Qt
#include <QX11Info>
#include <QCoreApplication>
#include <QGuiApplication>
#include <QScreen>
#include <QSocketNotifier>
#include <QDebug>
#include <KRandom>
// pipewire
#include <pipewire/pipewire.h>
#include <pipewire/global.h>
#include <spa/param/video/format-utils.h>
#include <spa/param/format-utils.h>
#include <spa/param/props.h>
#include <spa/lib/debug.h>
#include <limits.h>

#include "pw_framebuffer.h"
#include "xdp_dbus_interface.h"

static const uint MIN_SUPPORTED_XDP_KDE_SC_VERSION = 1;

/**
 * @brief The PwType class - helper class to contain pointers to raw C pipewire media mappings
 */
class PwType {
public:
    spa_type_media_type media_type;
    spa_type_media_subtype media_subtype;
    spa_type_format_video format_video;
    spa_type_video_format video_format;
};

/**
 * @brief The PWFrameBuffer::Private class - private counterpart of PWFramebuffer class. This is the entity where
 *        whole logic resides, for more info search for "d-pointer pattern" information.
 */
class PWFrameBuffer::Private {
public:
    Private(PWFrameBuffer *q);
    ~Private();

private:
    friend class PWFrameBuffer;

    static void onStateChanged(void *data, pw_remote_state old, pw_remote_state state, const char *error);
    static void onStreamStateChanged(void *data, pw_stream_state old, pw_stream_state state, const char *error_message);
    static void onStreamFormatChanged(void *data, struct spa_pod *format);
    static void onNewBuffer(void *data, uint32_t id);

    void initWayland();
    void initDbus();
    void initPw();
    void initializePwTypes();

    // dbus handling
    void handleSessionCreated(quint32 &code, QVariantMap &results);
    void handleSourcesSelected(quint32 &code, QVariantMap &results);
    void handleScreencastStarted(quint32 &code, QVariantMap &results);

    // pw handling
    void processPwEvents();
    void createReceivingStream();
    void handleFrame(spa_buffer *buf);

    // link to public interface
    PWFrameBuffer *q;

    // pipewire stuff
    pw_core *pwCore = nullptr;
    pw_type *pwCoreType = nullptr;
    pw_remote *pwRemote = nullptr;
    pw_stream *pwStream = nullptr;
    pw_loop *pwLoop = nullptr;
    QScopedPointer<PwType> pwType;

    // event handlers
    pw_remote_events pwRemoteEvents = {};
    pw_stream_events pwStreamEvents = {};

    // wayland-like listeners
    // ...of events that happen in pipewire server
    spa_hook remoteListener = {};
    // ...of events that happen with the stream we consume
    spa_hook streamListener = {};

    // negotiated video format
    QScopedPointer<spa_video_info_raw> videoFormat;

    // listens on pipewire socket
    QScopedPointer<QSocketNotifier> socketNotifier;

    // requests a session from XDG Desktop Portal
    // auto-generated and compiled from xdp_dbus_interface.xml file
    QScopedPointer<OrgFreedesktopPortalScreenCastInterface> dbusXdpService;

    // XDP screencast session handle
    QDBusObjectPath sessionPath;
    // Pipewire file descriptor
    QDBusUnixFileDescriptor pipewireFd;

    // counters for dbus exchange
    quint32 requestCounter = 0;
    quint32 sessionCounter = 0;

    // screen geometry holder
    struct {
        quint32 width;
        quint32 height;
    } screenGeometry;

    // real image with allocated memory which poses as a destination when we get a buffer from pipewire
    // and as source when we pass the frame back to protocol
    QImage fbImage;

    // sanity indicator
    bool isValid = true;
};

PWFrameBuffer::Private::Private(PWFrameBuffer *q) : q(q)
{
    // initialize event handlers, remote end and stream-related
    pwRemoteEvents.version = PW_VERSION_REMOTE_EVENTS;
    pwRemoteEvents.state_changed = &onStateChanged;

    pwStreamEvents.version = PW_VERSION_STREAM_EVENTS;
    pwStreamEvents.state_changed = &onStreamStateChanged;
    pwStreamEvents.format_changed = &onStreamFormatChanged;
    pwStreamEvents.new_buffer = &onNewBuffer;
}

/**
 * @brief PWFrameBuffer::Private::initWayland - initializes screen info and Wayland connectivity.
 *        For now just grabs first available screen and uses its dimensions for framebuffer.
 */
void PWFrameBuffer::Private::initWayland()
{
    qInfo() << "Initializing screen info";
    auto screen = qApp->screens().at(0);
    auto screenSize = screen->geometry();
    screenGeometry.width = static_cast<quint32>(screenSize.width());
    screenGeometry.height = static_cast<quint32>(screenSize.height());
    fbImage = QImage(screenSize.width(), screenSize.height(), QImage::Format_RGBX8888);
}

/**
 * @brief PWFrameBuffer::Private::initDbus - initialize D-Bus connectivity with XDG Desktop Portal.
 *        Based on XDG_CURRENT_DESKTOP environment variable it will give us implementation that we need,
 *        in case of KDE it is xdg-desktop-portal-kde binary.
 */
void PWFrameBuffer::Private::initDbus()
{
    qInfo() << "Initializing D-Bus connectivity with XDG Desktop Portal";
    dbusXdpService.reset(new OrgFreedesktopPortalScreenCastInterface(QLatin1String("org.freedesktop.portal.Desktop"),
                                                                     QLatin1String("/org/freedesktop/portal/desktop"),
                                                                     QDBusConnection::sessionBus()));
    if (!dbusXdpService->isValid()) {
        qWarning("Can't find XDG Portal screencast interface");
        isValid = false;
        return;
    }

    auto version = dbusXdpService->version();
    if (version < MIN_SUPPORTED_XDP_KDE_SC_VERSION) {
        qWarning() << "Unsupported XDG Portal screencast interface version:" << version;
        isValid = false;
        return;
    }

    // create session
    auto sessionParameters = QVariantMap {
        { QLatin1String("session_handle_token"), QString::number(sessionCounter++) },
        { QLatin1String("handle_token"), QString::number(requestCounter++) }
    };
    auto sessionReply = dbusXdpService->CreateSession(sessionParameters);
    sessionReply.waitForFinished();
    if (!sessionReply.isValid()) {
        qWarning("Couldn't initialize XDP-KDE screencast session");
        isValid = false;
        return;
    }

    qInfo() << "DBus session created: " << sessionReply.value().path();
    QDBusConnection::sessionBus().connect(QString(),
                                          sessionReply.value().path(),
                                          QLatin1String("org.freedesktop.portal.Request"),
                                          QLatin1String("Response"),
                                          this->q,
                                          SLOT(handleXdpSessionCreated(uint, QVariantMap)));
}

void PWFrameBuffer::handleXdpSessionCreated(quint32 code, QVariantMap results)
{
    d->handleSessionCreated(code, results);
}

/**
 * @brief PWFrameBuffer::Private::handleSessionCreated - handle creation of ScreenCast session.
 *        XDG Portal answers with session path if it was able to successfully create the screencast.
 *
 * @param code return code for dbus call. Zero is success, non-zero means error
 * @param results map with results of call.
 */
void PWFrameBuffer::Private::handleSessionCreated(quint32 &code, QVariantMap &results)
{
    if (code != 0) {
        qWarning() << "Failed to create session: " << code;
        isValid = false;
        return;
    }

    sessionPath = QDBusObjectPath(results.value(QLatin1String("session_handle")).toString());

    // select sources for the session
    auto selectionOptions = QVariantMap {
        { QLatin1String("types"), 1u }, // only MONITOR is supported
        { QLatin1String("multiple"), false },
        { QLatin1String("handle_token"), QString::number(requestCounter++) }
    };
    auto selectorReply = dbusXdpService->SelectSources(sessionPath, selectionOptions);
    selectorReply.waitForFinished();
    if (!selectorReply.isValid()) {
        qWarning() << "Couldn't select sources for the screen-casting session";
        isValid = false;
        return;
    }
    QDBusConnection::sessionBus().connect(QString(),
                                          selectorReply.value().path(),
                                          QLatin1String("org.freedesktop.portal.Request"),
                                          QLatin1String("Response"),
                                          this->q,
                                          SLOT(handleXdpSourcesSelected(uint, QVariantMap)));
}

void PWFrameBuffer::handleXdpSourcesSelected(quint32 code, QVariantMap results)
{
    d->handleSourcesSelected(code, results);
}

/**
 * @brief PWFrameBuffer::Private::handleSourcesSelected - handle Screencast sources selection.
 *        XDG Portal shows a dialog at this point which allows you to select monitor from the list.
 *        This function is called after you make a selection.
 *
 * @param code return code for dbus call. Zero is success, non-zero means error
 * @param results map with results of call.
 */
void PWFrameBuffer::Private::handleSourcesSelected(quint32 &code, QVariantMap &)
{
    if (code != 0) {
        qWarning() << "Failed to select sources: " << code;
        isValid = false;
        return;
    }

    // start session
    auto startParameters = QVariantMap {
        { QLatin1String("handle_token"), QString::number(requestCounter++) }
    };
    auto startReply = dbusXdpService->Start(sessionPath, QString(), startParameters);
    startReply.waitForFinished();
    QDBusConnection::sessionBus().connect(QString(),
                                          startReply.value().path(),
                                          QLatin1String("org.freedesktop.portal.Request"),
                                          QLatin1String("Response"),
                                          this->q,
                                          SLOT(handleXdpScreenCastStarted(uint, QVariantMap)));
}


void PWFrameBuffer::handleXdpScreenCastStarted(quint32 code, QVariantMap results)
{
    d->handleScreencastStarted(code, results);
}

/**
 * @brief PWFrameBuffer::Private::handleScreencastStarted - handle Screencast start.
 *        At this point there shall be ready pipewire stream to consume.
 *
 * @param code return code for dbus call. Zero is success, non-zero means error
 * @param results map with results of call.
 */
void PWFrameBuffer::Private::handleScreencastStarted(quint32 &code, QVariantMap &results)
{
    if (code != 0) {
        qWarning() << "Failed to start screencast: " << code;
        isValid = false;
        return;
    }

    // there should be only one stream
    auto streams = results.value("streams");
    if (streams.isNull()) {
        // maybe we should check deeper with qdbus_cast but this suffices for now
        qWarning() << "Failed to get screencast streams";
        isValid = false;
        return;
    }

    auto streamReply = dbusXdpService->OpenPipeWireRemote(sessionPath, QVariantMap());
    streamReply.waitForFinished();
    if (!streamReply.isValid()) {
        qWarning() << "Couldn't open pipewire remote for the screen-casting session";
        isValid = false;
        return;
    }

    pipewireFd = streamReply.value();
    if (!pipewireFd.isValid()) {
        qWarning() << "Couldn't get pipewire connection file descriptor";
        isValid = false;
        return;
    }
    initPw();
}

/**
 * @brief PWFrameBuffer::Private::initPw - initialize Pipewire socket connectivity.
 *        pipewireFd should be pointing to existing file descriptor that was passed by D-Bus at this point.
 */
void PWFrameBuffer::Private::initPw() {
    qInfo() << "Initializing Pipewire connectivity";

    // init pipewire (required)
    pw_init(nullptr, nullptr); // args are not used anyways

    // initialize our source
    pwLoop = pw_loop_new(nullptr);
    socketNotifier.reset(new QSocketNotifier(pw_loop_get_fd(pwLoop), QSocketNotifier::Read));
    QObject::connect(socketNotifier.data(), &QSocketNotifier::activated, this->q, &PWFrameBuffer::processPwEvents);

    // create PipeWire core object (required)
    pwCore = pw_core_new(pwLoop, nullptr);
    pwCoreType = pw_core_get_type(pwCore);

    // pw_remote should be initialized before type maps or connection error will happen
    pwRemote = pw_remote_new(pwCore, nullptr, 0);

    // init type maps
    initializePwTypes();

    // init PipeWire remote, add listener to handle events
    pw_remote_add_listener(pwRemote, &remoteListener, &pwRemoteEvents, this);
    pw_remote_connect_fd(pwRemote, pipewireFd.fileDescriptor());
}

/**
 * @brief PWFrameBuffer::Private::initializePwTypes - helper method to initialize and map all needed
 *        Pipewire types from core to type structure.
 */
void PWFrameBuffer::Private::initializePwTypes()
{
    // raw C-like PipeWire type map
    auto map = pwCoreType->map;

    pwType.reset(new PwType);
    spa_type_media_type_map(map, &pwType->media_type);
    spa_type_media_subtype_map(map, &pwType->media_subtype);
    spa_type_format_video_map(map, &pwType->format_video);
    spa_type_video_format_map(map, &pwType->video_format);

    // must be called after type system is mapped
    // calling it before causes unpredictable memory corruption and errors
    spa_debug_set_type_map(pwCoreType->map);
}

/**
 * @brief PWFrameBuffer::Private::onStateChanged - global state tracking for pipewire connection
 * @param data pointer that you have set in pw_remote_add_listener call's last argument
 * @param state new state that connection has changed to
 * @param error optional error message, is set to non-null if state is error
 */
void PWFrameBuffer::Private::onStateChanged(void *data, pw_remote_state /*old*/, pw_remote_state state, const char *error)
{
    qInfo() << "remote state: " << pw_remote_state_as_string(state);

    PWFrameBuffer::Private *d = static_cast<PWFrameBuffer::Private*>(data);

    switch (state) {
    case PW_REMOTE_STATE_ERROR:
        qWarning() << "remote error: " << error;
        break;
    case PW_REMOTE_STATE_CONNECTED:
        d->createReceivingStream();
        break;
    default:
        qInfo() << "remote state: " << pw_remote_state_as_string(state);
        break;
    }
}

/**
 * @brief PWFrameBuffer::Private::onStreamStateChanged - called whenever stream state changes on pipewire server
 * @param data pointer that you have set in pw_stream_add_listener call's last argument
 * @param state new state that stream has changed to
 * @param error_message optional error message, is set to non-null if state is error
 */
void PWFrameBuffer::Private::onStreamStateChanged(void *data, pw_stream_state /*old*/, pw_stream_state state, const char *error_message)
{
    qInfo() << "Stream state changed: " << pw_stream_state_as_string(state);

    auto *d = static_cast<PWFrameBuffer::Private *>(data);

    switch (state) {
    case PW_STREAM_STATE_ERROR:
        qWarning() << "pipewire stream error: " << error_message;
        break;
    case PW_STREAM_STATE_CONFIGURE:
        pw_stream_set_active(d->pwStream, true);
        break;
    default:
        break;
    }
}

/**
 * @brief PWFrameBuffer::Private::onStreamFormatChanged - being executed after stream is set to active
 *        and after setup has been requested to connect to it. The actual video format is being negotiated here.
 * @param data pointer that you have set in pw_stream_add_listener call's last argument
 * @param format format that's being proposed
 */
void PWFrameBuffer::Private::onStreamFormatChanged(void *data, struct spa_pod *format)
{
    qInfo() << "Stream format changed";
    auto *d = static_cast<PWFrameBuffer::Private *>(data);

    const int bpp = 4;

    if (!format) {
        pw_stream_finish_format(d->pwStream, 0, nullptr, 0);
        return;
    }

    d->videoFormat.reset(new spa_video_info_raw);
    spa_format_video_raw_parse(format, d->videoFormat.data(), &d->pwType->format_video);

    auto width = d->videoFormat->size.width;
    auto height = d->videoFormat->size.height;
    auto stride = SPA_ROUND_UP_N(width * bpp, 4);
    auto size = height * stride;

    uint8_t buffer[1024];
    auto builder = spa_pod_builder {buffer, sizeof(buffer)};

    // setup buffers and meta header for new format
    struct spa_pod *params[2];
    params[0] = reinterpret_cast<spa_pod *>(spa_pod_builder_object(&builder,
             d->pwCoreType->param.idBuffers, d->pwCoreType->param_buffers.Buffers,
        ":", d->pwCoreType->param_buffers.size, "i", size,
        ":", d->pwCoreType->param_buffers.stride, "i", stride,
        ":", d->pwCoreType->param_buffers.buffers, "iru", 8, SPA_POD_PROP_MIN_MAX(1, 32),
        ":", d->pwCoreType->param_buffers.align, "i", 16));
    params[1] = reinterpret_cast<spa_pod *>(spa_pod_builder_object(&builder,
             d->pwCoreType->param.idMeta, d->pwCoreType->param_meta.Meta,
        ":", d->pwCoreType->param_meta.type, "I", d->pwCoreType->meta.Header,
        ":", d->pwCoreType->param_meta.size, "i", sizeof(struct spa_meta_header)));

    pw_stream_finish_format(d->pwStream, 0, params, 2);
}

/**
 * @brief PWFrameBuffer::Private::onNewBuffer - called when new buffer is available in pipewire stream
 * @param data pointer that you have set in pw_stream_add_listener call's last argument
 * @param id
 */
void PWFrameBuffer::Private::onNewBuffer(void *data, uint32_t id)
{
    qDebug() << "New buffer received" << id;
    auto *d = static_cast<PWFrameBuffer::Private *>(data);

    auto buf = pw_stream_peek_buffer(d->pwStream, id);
    d->handleFrame(buf);

    pw_stream_recycle_buffer(d->pwStream, id);
}

void PWFrameBuffer::Private::handleFrame(spa_buffer *buf)
{
    auto mapLength = buf->datas[0].maxsize + buf->datas[0].mapoffset;

    void *mapped; // full length of mapped data
    void *src; // real pixel data in this buffer
    if (buf->datas[0].type == pwCoreType->data.MemFd || buf->datas[0].type == pwCoreType->data.DmaBuf) {
        mapped = mmap(nullptr, mapLength, PROT_READ, MAP_PRIVATE, buf->datas[0].fd, 0);
        src = SPA_MEMBER(mapped, buf->datas[0].mapoffset, void);
    } else if (buf->datas[0].type == pwCoreType->data.MemPtr) {
        mapped = nullptr;
        src = buf->datas[0].data;
    } else {
        qWarning() << "Got unsupported buffer type" << buf->datas[0].type;
        return;
    }

    qint32 srcStride = buf->datas[0].chunk->stride;
    if (srcStride != q->paddedWidth()) {
        qWarning() << "Got buffer with stride different from screen stride" << srcStride << "!=" << q->paddedWidth();
        return;
    }

    fbImage.bits();
    q->tiles.append(fbImage.rect());
    std::memcpy(fbImage.bits(), src, buf->datas[0].maxsize);

    if (mapped)
        munmap(mapped, mapLength);
}

/**
 * @brief PWFrameBuffer::Private::processPwEvents - called when Pipewire socket notifies there's
 *        data to process by remote interface.
 */
void PWFrameBuffer::Private::processPwEvents() {
    qDebug() << "Iterating over pipewire loop...";

    int result = pw_loop_iterate(pwLoop, 0);
    if (result < 0) {
        qWarning() << "Failed to iterate over pipewire loop: " << spa_strerror(result);
    }
}

/**
 * @brief PWFrameBuffer::Private::createReceivingStream - create a stream that will consume Pipewire buffers
 *        and copy the framebuffer to the existing image that we track. The state of the stream and configuration
 *        are later handled by the corresponding listener.
 */
void PWFrameBuffer::Private::createReceivingStream()
{
    auto pwScreenBounds = spa_rectangle {screenGeometry.width, screenGeometry.height};

    auto pwFramerate = spa_fraction {25, 1};
    auto pwFramerateMin = spa_fraction {0, 1};
    auto pwFramerateMax = spa_fraction {60, 1};

    auto reuseProps = pw_properties_new("pipewire.client.reuse", "1", nullptr); // null marks end of varargs
    pwStream = pw_stream_new(pwRemote, "krfb-fb-consume-stream", reuseProps);

    uint8_t buffer[1024] = {};
    const spa_pod *params[1];
    auto builder = spa_pod_builder{buffer, sizeof(buffer)};
    params[0] = reinterpret_cast<spa_pod *>(spa_pod_builder_object(&builder,
          pwCoreType->param.idEnumFormat, pwCoreType->spa_format,
          "I", pwType->media_type.video,
          "I", pwType->media_subtype.raw,
          ":", pwType->format_video.format, "I", pwType->video_format.RGBx,
          ":", pwType->format_video.size, "R", &pwScreenBounds,
          ":", pwType->format_video.framerate, "F", &pwFramerateMin,
          ":", pwType->format_video.max_framerate, "Fr", &pwFramerate, 2, &pwFramerateMin, &pwFramerateMax));
    spa_debug_pod(params[0], SPA_DEBUG_FLAG_FORMAT);

    pw_stream_add_listener(pwStream, &streamListener, &pwStreamEvents, this);
    auto flags = static_cast<pw_stream_flags>(PW_STREAM_FLAG_AUTOCONNECT | PW_STREAM_FLAG_INACTIVE);
    if (pw_stream_connect(pwStream, PW_DIRECTION_INPUT, nullptr, flags, params, 1) != 0) {
        qWarning() << "Could not connect receiving stream";
        isValid = false;
    }
}

PWFrameBuffer::Private::~Private()
{
    if (pwStream) {
        pw_stream_disconnect(pwStream);
        pw_stream_destroy(pwStream);
    }
    if (pwRemote) {
        pw_remote_disconnect(pwRemote);
    }

    if (pwCore)
        pw_core_destroy(pwCore);

    if (pwLoop) {
        pw_loop_leave(pwLoop);
        pw_loop_destroy(pwLoop);
    }
}

PWFrameBuffer::PWFrameBuffer(WId winid, QObject *parent)
    : FrameBuffer (winid, parent),
      d(new Private(this))
{
    // D-Bus is most important in init chain, no toys for us if something is wrong with XDP
    // PipeWire connectivity is initialized after D-Bus session is started
    d->initDbus();

    // connect to Wayland and PipeWire sockets
    d->initWayland();

    // framebuffer from public interface will point directly to image data
    fb = reinterpret_cast<char *>(d->fbImage.bits());
}

PWFrameBuffer::~PWFrameBuffer()
{
    fb = nullptr;
}

void PWFrameBuffer::processPwEvents()
{
    d->processPwEvents();
}

int PWFrameBuffer::depth()
{
    return 32;
}

int PWFrameBuffer::height()
{
    return static_cast<qint32>(d->screenGeometry.height);
}

int PWFrameBuffer::width()
{
    return static_cast<qint32>(d->screenGeometry.width);
}

void PWFrameBuffer::getServerFormat(rfbPixelFormat &format)
{
    format.bitsPerPixel = 32;
    format.depth = 32;
    format.trueColour = true;
    format.bigEndian = false;
}

void PWFrameBuffer::startMonitor()
{

}

void PWFrameBuffer::stopMonitor()
{

}

bool PWFrameBuffer::isValid() const
{
    return d->isValid;
}