GStreamer基礎教程11 – 與QT集成
- 2019 年 10 月 31 日
- 筆記
摘要
通常我們的播放引擎需要和GUI進行集成，在使用GStreamer時，GStreamre會負責媒體的播放及控制，GUI會負責處理用戶的交互操作以及創建顯示的窗口。本例中我們將結合QT介紹如何指定GStreamer將影片輸出到指定窗口，以及如何利用GStreamer上報的資訊去更新GUI。
與GUI集成
我們知道與GUI集成有兩個方面需要注意：
- 顯示窗口的管理。
由於顯示窗口通常由GUI框架創建，所以我們需要將具體的窗口資訊告訴GStreamer。由於各個平台使用不同的方式傳遞窗口句柄，GStreamer提供了一個抽象介面（GstVideoOverlay），用於屏蔽平台的差異，我們可以直接將GUI創建的窗口ID傳遞給GStreamer。
- GUI介面的更新
大多數GUI框架都需要在主執行緒中去做UI的刷新操作，但GStreamer內部可能會創建多個執行緒，這就需要通過GstBus及GUI自帶的通訊機制將所有GStreamer產生的消息傳遞到GUI主執行緒，再由GUI主執行緒對介面進行刷新。
下面我們將以QT為例來了解如何處理GStreamer與GUI框架的集成。
示例程式碼
qtoverlay.h
#ifndef _QTOVERLAY_ #define _QTOVERLAY_ #include <gst/gst.h> #include <QWidget> #include <QPushButton> #include <QHBoxLayout> #include <QVBoxLayout> #include <QSlider> #include <QTimer> class PlayerWindow : public QWidget { Q_OBJECT public: PlayerWindow(GstElement *p); WId getVideoWId() const ; static gboolean postGstMessage(GstBus * bus, GstMessage * message, gpointer user_data); private slots: void onPlayClicked() ; void onPauseClicked() ; void onStopClicked() ; void onAlbumAvaiable(const QString &album); void onState(GstState st); void refreshSlider(); void onSeek(); void onEos(); signals: void sigAlbum(const QString &album); void sigState(GstState st); void sigEos(); private: GstElement *pipeline; QPushButton *playBt; QPushButton *pauseBt; QPushButton *stopBt; QWidget *videoWindow; QSlider *slider; QHBoxLayout *buttonLayout; QVBoxLayout *playerLayout; QTimer *timer; GstState state; gint64 totalDuration; }; #endif
View Code
qtoverlay.cpp
#include <gst/video/videooverlay.h> #include <QApplication> #include "qtoverlay.h" PlayerWindow::PlayerWindow(GstElement *p) :pipeline(p) ,state(GST_STATE_NULL) ,totalDuration(GST_CLOCK_TIME_NONE) { playBt = new QPushButton("Play"); pauseBt = new QPushButton("Pause"); stopBt = new QPushButton("Stop"); videoWindow = new QWidget(); slider = new QSlider(Qt::Horizontal); timer = new QTimer(); connect(playBt, SIGNAL(clicked()), this, SLOT(onPlayClicked())); connect(pauseBt, SIGNAL(clicked()), this, SLOT(onPauseClicked())); connect(stopBt, SIGNAL(clicked()), this, SLOT(onStopClicked())); connect(slider, SIGNAL(sliderReleased()), this, SLOT(onSeek())); buttonLayout = new QHBoxLayout; buttonLayout->addWidget(playBt); buttonLayout->addWidget(pauseBt); buttonLayout->addWidget(stopBt); buttonLayout->addWidget(slider); playerLayout = new QVBoxLayout; playerLayout->addWidget(videoWindow); playerLayout->addLayout(buttonLayout); this->setLayout(playerLayout); connect(timer, SIGNAL(timeout()), this, SLOT(refreshSlider())); connect(this, SIGNAL(sigAlbum(QString)), this, SLOT(onAlbumAvaiable(QString))); connect(this, SIGNAL(sigState(GstState)), this, SLOT(onState(GstState))); connect(this, SIGNAL(sigEos()), this, SLOT(onEos())); } WId PlayerWindow::getVideoWId() const { return videoWindow->winId(); } void PlayerWindow::onPlayClicked() { GstState st = GST_STATE_NULL; gst_element_get_state (pipeline, &st, NULL, GST_CLOCK_TIME_NONE); if (st < GST_STATE_PAUSED) { // Pipeline stopped, we need set overlay again GstElement *vsink = gst_element_factory_make ("ximagesink", "vsink"); g_object_set(GST_OBJECT(pipeline), "video-sink", vsink, NULL); WId xwinid = getVideoWId(); gst_video_overlay_set_window_handle (GST_VIDEO_OVERLAY (vsink), xwinid); } gst_element_set_state (pipeline, GST_STATE_PLAYING); } void PlayerWindow::onPauseClicked() { gst_element_set_state (pipeline, GST_STATE_PAUSED); } void PlayerWindow::onStopClicked() { gst_element_set_state (pipeline, GST_STATE_NULL); } void PlayerWindow::onAlbumAvaiable(const QString &album) { setWindowTitle(album); } void PlayerWindow::onState(GstState st) { if (state != st) { state = st; if (state == GST_STATE_PLAYING){ timer->start(1000); } if (state < GST_STATE_PAUSED){ timer->stop(); } } } void PlayerWindow::refreshSlider() { gint64 current = GST_CLOCK_TIME_NONE; if (state == GST_STATE_PLAYING) { if (!GST_CLOCK_TIME_IS_VALID(totalDuration)) { if (gst_element_query_duration (pipeline, GST_FORMAT_TIME, &totalDuration)) { slider->setRange(0, totalDuration/GST_SECOND); } } if (gst_element_query_position (pipeline, GST_FORMAT_TIME, ¤t)) { g_print("%ld / %ldn", current/GST_SECOND, totalDuration/GST_SECOND); slider->setValue(current/GST_SECOND); } } } void PlayerWindow::onSeek() { gint64 pos = slider->sliderPosition(); g_print("seek: %ldn", pos); gst_element_seek_simple (pipeline, GST_FORMAT_TIME, GST_SEEK_FLAG_FLUSH , pos * GST_SECOND); } void PlayerWindow::onEos() { gst_element_set_state (pipeline, GST_STATE_NULL); } gboolean PlayerWindow::postGstMessage(GstBus * bus, GstMessage * message, gpointer user_data) { PlayerWindow *pw = NULL; if (user_data) { pw = reinterpret_cast<PlayerWindow*>(user_data); } switch (GST_MESSAGE_TYPE(message)) { case GST_MESSAGE_STATE_CHANGED: { GstState old_state, new_state, pending_state; gst_message_parse_state_changed (message, &old_state, &new_state, &pending_state); pw->sigState(new_state); break; } case GST_MESSAGE_TAG: { GstTagList *tags = NULL; gst_message_parse_tag(message, &tags); gchar *album= NULL; if (gst_tag_list_get_string(tags, GST_TAG_ALBUM, &album)) { pw->sigAlbum(album); g_free(album); } gst_tag_list_unref(tags); break; } case GST_MESSAGE_EOS: { pw->sigEos(); break; } default: break; } return TRUE; } int main(int argc, char *argv[]) { gst_init (&argc, &argv); QApplication app(argc, argv); app.connect(&app, SIGNAL(lastWindowClosed()), &app, SLOT(quit ())); // prepare the pipeline GstElement *pipeline = gst_parse_launch ("playbin uri=file:///home/john/video/sintel_trailer-480p.webm", NULL); // prepare the ui PlayerWindow *window = new PlayerWindow(pipeline); window->resize(900, 600); window->show(); // seg window id to gstreamer GstElement *vsink = gst_element_factory_make ("ximagesink", "vsink"); WId xwinid = window->getVideoWId(); gst_video_overlay_set_window_handle (GST_VIDEO_OVERLAY (vsink), xwinid); g_object_set(GST_OBJECT(pipeline), "video-sink", vsink, NULL); // connect to interesting signals GstBus *bus = gst_element_get_bus(pipeline); gst_bus_add_watch(bus, &PlayerWindow::postGstMessage, window); gst_object_unref(bus); // run the pipeline GstStateChangeReturn sret = gst_element_set_state (pipeline, GST_STATE_PLAYING); if (sret == GST_STATE_CHANGE_FAILURE) { gst_element_set_state (pipeline, GST_STATE_NULL); gst_object_unref (pipeline); // Exit application QTimer::singleShot(0, QApplication::activeWindow(), SLOT(quit())); } int ret = app.exec(); window->hide(); gst_element_set_state (pipeline, GST_STATE_NULL); gst_object_unref (pipeline); return ret; }
qtoverlay.pro
QT += core gui widgets TARGET = qtoverlay INCLUDEPATH += /usr/include/glib-2.0 INCLUDEPATH += /usr/lib/x86_64-linux-gnu/glib-2.0/include INCLUDEPATH += /usr/include/gstreamer-1.0 INCLUDEPATH += /usr/lib/x86_64-linux-gnu/gstreamer-1.0/include LIBS += -lgstreamer-1.0 -lgobject-2.0 -lglib-2.0 -lgstvideo-1.0 SOURCES += qtoverlay.cpp HEADERS += qtoverlay.h
分別保存以上內容到各個文件，執行下列命令即可得到可執行程式。如果找不到頭文件及庫文件，需要根據實際路徑修改qtoverlay.pro文件中的內容。
qmake -o Makefile qtoverlay.pro make
源碼分析
// prepare the pipeline GstElement *pipeline = gst_parse_launch ("playbin uri=file:///home/jleng/video/sintel_trailer-480p.webm", NULL); // prepare the ui PlayerWindow *window = new PlayerWindow(pipeline); window->resize(900, 600); window->show();
在main函數中對GStreamer進行初始化及創建了QT的應用對象後，構造了Pipline，構造GUI窗口對象。在PlayerWindow的構造函數中初始化按鈕及窗口，同時創建定時刷新進度條的Timer。
// seg window id to gstreamer GstElement *vsink = gst_element_factory_make ("ximagesink", "vsink"); WId xwinid = window->getVideoWId(); gst_video_overlay_set_window_handle (GST_VIDEO_OVERLAY (vsink), xwinid); g_object_set(GST_OBJECT(pipeline), "video-sink", vsink, NULL); ... gst_bus_add_watch(bus, &PlayerWindow::postGstMessage, window); ... GstStateChangeReturn sret = gst_element_set_state (pipeline, GST_STATE_PLAYING); ... int ret = app.exec(); ...
接著我們單獨創建了ximagesink用於影片渲染，同時我們將Qt創建的影片窗口ID設置給GStreamer，讓GStreamer得到渲染的窗口ID，接著使用g_object_set()將自定義的Sink通過“video-sink”屬性設置到playbin中。
同時，我們設置了GStreamer的消息處理函數，所有的消息都會在postGstMessage函數中被轉發。為了後續調用GUI對象中的介面，我們需要將GUI窗口指針作為user-data，在postGstMessage中再轉換為GUI對象。
接著設置Pipeline的狀態為PLAYING開始播放。
最後調用GUI框架的事件循環，exec()會一直執行，直到關閉窗口。
由於GStreamer的GstBus會默認使用GLib的主循環及事件處理機制，所以必須要保證GLi默認的MainLoop在某個執行緒中運行。在本例中，Qt在Linux下會自動使用GLib的主循環，所以我們無需額外進行處理。
gboolean PlayerWindow::postGstMessage(GstBus * bus, GstMessage * message, gpointer user_data) { PlayerWindow *pw = NULL; if (user_data) { pw = reinterpret_cast<PlayerWindow*>(user_data); } switch (GST_MESSAGE_TYPE(message)) { case GST_MESSAGE_STATE_CHANGED: { GstState old_state, new_state, pending_state; gst_message_parse_state_changed (message, &old_state, &new_state, &pending_state); pw->sigState(new_state); break; } case GST_MESSAGE_TAG: { GstTagList *tags = NULL; gst_message_parse_tag(message, &tags); gchar *album= NULL; if (gst_tag_list_get_string(tags, GST_TAG_ALBUM, &album)) { pw->sigAlbum(album); g_free(album); } gst_tag_list_unref(tags); break; } case GST_MESSAGE_EOS: { pw->sigEos(); break; } default: break; } return TRUE; }
我們在轉換後GUI對象後，再根據消息類型進行處理。在postGstMessage中我們沒有直接更新GUI，因為GStreamer的Bus處理執行緒與GUI主執行緒可能為不同執行緒，直接更新GUI會出錯或無效。因此利用Qt的signal-slot機制在相應的槽函數中就行GUI資訊的更新。這裡只處理了3種消息STATE_CHANGED（狀態變化），TAG（媒體元數據及編碼資訊），EOS（播放結束），GStreamer所支援的消息可查看官方文檔GstMessage。
void PlayerWindow::onPlayClicked() { GstState st = GST_STATE_NULL; gst_element_get_state (pipeline, &st, NULL, GST_CLOCK_TIME_NONE); if (st < GST_STATE_PAUSED) { // Pipeline stopped, we need set overlay again GstElement *vsink = gst_element_factory_make ("ximagesink", "vsink"); g_object_set(GST_OBJECT(pipeline), "video-sink", vsink, NULL); WId xwinid = getVideoWId(); gst_video_overlay_set_window_handle (GST_VIDEO_OVERLAY (vsink), xwinid); } gst_element_set_state (pipeline, GST_STATE_PLAYING); }
當點擊Play按鈕時，onPlayClicked函數會被調用，我們在此直接調用GStreamer的介面設置Pipeline的狀態。當播放結束或點擊Stop時，GStreamer會在狀態切換到NULL時釋放所有資源，所以我們在此需要重新設置playbin的vido-sink，並指定影片輸出窗口。
Pause，Stop的處理類似，直接調用gst_element_set_state ()將Pipeline設置為相應狀態。
void PlayerWindow::refreshSlider() { gint64 current = GST_CLOCK_TIME_NONE; if (state == GST_STATE_PLAYING) { if (!GST_CLOCK_TIME_IS_VALID(totalDuration)) { if (gst_element_query_duration (pipeline, GST_FORMAT_TIME, &totalDuration)) { slider->setRange(0, totalDuration/GST_SECOND); } } if (gst_element_query_position (pipeline, GST_FORMAT_TIME, ¤t)) { g_print("%ld / %ldn", current/GST_SECOND, totalDuration/GST_SECOND); slider->setValue(current/GST_SECOND); } } } void PlayerWindow::onSeek() { gint64 pos = slider->sliderPosition(); g_print("seek: %ldn", pos); gst_element_seek_simple (pipeline, GST_FORMAT_TIME, GST_SEEK_FLAG_FLUSH , pos * GST_SECOND); }
我們在構造函數中創建了Timer用於每秒刷新進度條，在refreshSlider被調用時，我們通過gst_element_query_duration() 和gst_element_query_position ()得到文件的總時間和當前時間，並刷新進度條。由於GStreamer返回時間單位為納秒，所以我們需要通過GST_SECOND將其轉換為秒用於時間顯示。
我們同樣處理了用戶的Seek操作，在拉動進度條到某個位置時，獲取Seek的位置，調用gst_element_seek_simple ()跳轉到指定位置。我們不用關心對GStreamer的調用是處於哪個執行緒，GStreamer內部會自動進行處理。
總結
通過本文，我們學習到：
- 如何使用gst_video_overlay_set_window_handle ()將GUI的窗口句柄傳遞給GStremaer。
- 如何使用訊號槽傳遞消息到GUI主執行緒。
- 如何使用Timer定時刷新GUI。
引用
https://gstreamer.freedesktop.org/documentation/video/gstvideooverlay.html?gi-language=c
https://gstreamer.freedesktop.org/documentation/tutorials/basic/toolkit-integration.html?gi-language=c
https://doc.qt.io/qt-5/qmake-manual.html
