30 #include <gst/pbutils/missing-plugins.h>
32 #include <hybris/media/surface_texture_client_hybris.h>
33 #include <hybris/media/media_codec_layer.h>
35 #include <QMimeDatabase>
39 #include <sys/socket.h>
46 static const char *PULSE_SINK =
"pulsesink";
47 static const char *HYBRIS_SINK =
"hybrissink";
48 static const char *MIR_SINK =
"mirsink";
53 void gstreamer::Playbin::setup_video_sink_for_buffer_streaming()
55 IGBPWrapperHybris igbp;
56 SurfaceTextureClientHybris stc;
58 GstStructure *structure;
61 case lomiri::MediaHubService::AVBackend::Backend::hybris:
64 igbp = decoding_service_get_igraphicbufferproducer();
65 stc = surface_texture_client_create_by_igbp(igbp);
68 surface_texture_client_set_hardware_rendering(stc, TRUE);
70 context = gst_context_new(
"gst.mir.MirContext", TRUE);
71 structure = gst_context_writable_structure(context);
72 gst_structure_set(structure,
"gst_mir_context", G_TYPE_POINTER, stc, NULL);
75 gst_element_set_context(
pipeline, context);
77 case lomiri::MediaHubService::AVBackend::Backend::mir:
79 connect_to_consumer();
82 g_object_set (G_OBJECT (
video_sink),
"export-buffers", TRUE,
nullptr);
84 case lomiri::MediaHubService::AVBackend::Backend::none:
86 throw lomiri::MediaHubService::Player::Errors::
87 OutOfProcessBufferStreamingNotSupported{};
91 bool gstreamer::Playbin::is_supported_video_sink(
void)
const
93 if (video_sink_name == HYBRIS_SINK || video_sink_name == MIR_SINK)
107 static const std::string s{
"playbin"};
113 auto thiz =
static_cast<Playbin*
>(user_data);
121 if (user_data ==
nullptr)
124 static_cast<Playbin*
>(user_data)->setup_source(source);
132 gst_element_post_message(pipeline,
133 gst_message_new_application(GST_OBJECT(pipeline),
134 gst_structure_new_empty(
"streams-changed")));
138 : pipeline(gst_element_factory_make(
"playbin", pipeline_name().c_str())),
140 m_fileType(MEDIA_FILE_TYPE_NONE),
144 previous_position(0),
145 player_lifetime(media::Player::Lifetime::normal),
146 about_to_finish_handler_id(0),
147 source_setup_handler_id(0),
148 is_missing_audio_codec(
false),
149 is_missing_video_codec(
false),
152 current_new_state(GST_STATE_NULL),
158 throw std::runtime_error(
"Could not create pipeline for playbin.");
166 setup_pipeline_for_audio_video();
168 about_to_finish_handler_id = g_signal_connect(
171 G_CALLBACK(about_to_finish),
175 source_setup_handler_id = g_signal_connect(
178 G_CALLBACK(source_setup),
182 m_audioChangedHandlerId = g_signal_connect(
183 pipeline,
"audio-changed",
184 G_CALLBACK(streams_changed),
this);
186 m_videoChangedHandlerId = g_signal_connect(
187 pipeline,
"video-changed",
188 G_CALLBACK(streams_changed),
this);
212 print_refs(*
this,
"gstreamer::Playbin::~Playbin pipeline");
215 g_signal_handler_disconnect(pipeline, about_to_finish_handler_id);
216 g_signal_handler_disconnect(pipeline, source_setup_handler_id);
217 g_signal_handler_disconnect(pipeline, m_audioChangedHandlerId);
218 g_signal_handler_disconnect(pipeline, m_videoChangedHandlerId);
221 gst_object_unref(pipeline);
223 if (sock_consumer != -1) {
224 close(sock_consumer);
229 print_refs(*
this,
"gstreamer::Playbin::~Playbin pipeline");
241 if (player_lifetime != media::Player::Lifetime::resumable) {
249 const auto ret = gst_element_set_state(pipeline, GST_STATE_NULL);
252 case GST_STATE_CHANGE_FAILURE:
253 MH_WARNING(
"Failed to reset the pipeline state. Client reconnect may not function properly.");
255 case GST_STATE_CHANGE_NO_PREROLL:
256 case GST_STATE_CHANGE_SUCCESS:
257 case GST_STATE_CHANGE_ASYNC:
260 MH_WARNING(
"Failed to reset the pipeline state. Client reconnect may not function properly.");
262 setMediaFileType(MEDIA_FILE_TYPE_NONE);
263 is_missing_audio_codec =
false;
264 is_missing_video_codec =
false;
265 audio_stream_id = -1;
266 video_stream_id = -1;
267 if (sock_consumer != -1) {
268 close(sock_consumer);
273 void gstreamer::Playbin::process_missing_plugin_message(GstMessage *message)
275 gchar *desc = gst_missing_plugin_message_get_description(message);
279 const GstStructure *msg_data = gst_message_get_structure(message);
280 if (g_strcmp0(
"decoder", gst_structure_get_string(msg_data,
"type")) != 0)
284 if (!gst_structure_get(msg_data,
"detail", GST_TYPE_CAPS, &caps, NULL)) {
289 GstStructure *caps_data = gst_caps_get_structure(caps, 0);
295 const gchar *mime = gst_structure_get_name(caps_data);
296 if (strstr(mime,
"audio"))
297 is_missing_audio_codec =
true;
298 else if (strstr(mime,
"video"))
299 is_missing_video_codec =
true;
301 MH_ERROR(
"Missing decoder for %s", mime);
307 if (state.
new_state == GST_STATE_PAUSED ||
312 const QSize new_dimensions = get_video_dimensions();
313 Q_EMIT videoDimensionChanged(new_dimensions);
315 catch (
const std::exception& e)
317 MH_WARNING(
"Problem querying video dimensions: %s", e.what());
321 MH_WARNING(
"Problem querying video dimensions.");
328 const GstStructure *msg_data = gst_message_get_structure(message);
329 const gchar *struct_name = gst_structure_get_name(msg_data);
331 if (g_strcmp0(
"buffer-export-data", struct_name) == 0)
335 if (!gst_structure_get(msg_data,
336 "fd", G_TYPE_INT, &fd,
337 "width", G_TYPE_INT, &meta.
width,
338 "height", G_TYPE_INT, &meta.
height,
339 "fourcc", G_TYPE_INT, &meta.
fourcc,
340 "stride", G_TYPE_INT, &meta.
stride,
341 "offset", G_TYPE_INT, &meta.
offset,
344 MH_ERROR(
"Bad buffer-export-data message: mirsink version mismatch?");
348 send_buffer_data(fd, &meta,
sizeof meta);
350 else if (g_strcmp0(
"frame-ready", struct_name) == 0)
354 else if (g_strcmp0(
"streams-changed", struct_name) == 0)
356 updateMediaFileType();
360 MH_ERROR(
"Unknown GST_MESSAGE_ELEMENT with struct %s", struct_name);
366 switch (message.
type)
368 case GST_MESSAGE_ERROR:
371 case GST_MESSAGE_WARNING:
374 case GST_MESSAGE_INFO:
377 case GST_MESSAGE_STATE_CHANGED:
378 if (message.
source ==
"playbin") {
379 g_object_get(G_OBJECT(pipeline),
"current-audio", &audio_stream_id, NULL);
380 g_object_get(G_OBJECT(pipeline),
"current-video", &video_stream_id, NULL);
381 #ifdef DEBUG_GST_PIPELINE
382 MH_DEBUG(
"Dumping pipeline dot file");
383 GST_DEBUG_BIN_TO_DOT_FILE((GstBin*)pipeline, GST_DEBUG_GRAPH_SHOW_ALL,
"pipeline");
388 }
else if (message.
source ==
"video-sink") {
393 case GST_MESSAGE_APPLICATION:
394 case GST_MESSAGE_ELEMENT:
395 if (gst_is_missing_plugin_message(message.
message))
396 process_missing_plugin_message(message.
message);
398 process_message_element(message.
message);
400 case GST_MESSAGE_TAG:
403 if (gst_tag_list_get_string(message.
detail.
tag.
tag_list,
"image-orientation", &orientation))
406 Q_EMIT orientationChanged(orientation_lut(orientation));
407 g_free (orientation);
413 case GST_MESSAGE_ASYNC_DONE:
421 case GST_MESSAGE_EOS:
422 Q_EMIT endOfStream();
424 case GST_MESSAGE_BUFFERING:
440 g_object_get (pipeline,
"flags", &flags,
nullptr);
441 flags |= GST_PLAY_FLAG_AUDIO;
442 flags |= GST_PLAY_FLAG_VIDEO;
443 flags &= ~GST_PLAY_FLAG_TEXT;
444 g_object_set (pipeline,
"flags", flags,
nullptr);
446 const char *asink_name = ::getenv(
"CORE_MEDIA_SERVICE_AUDIO_SINK_NAME");
448 if (asink_name ==
nullptr)
449 asink_name = PULSE_SINK;
451 audio_sink = gst_element_factory_make (asink_name,
"audio-sink");
453 g_object_set (pipeline,
"audio-sink", audio_sink, NULL);
454 if (strcmp(asink_name,
"fakesink") == 0) {
455 g_object_set(audio_sink,
"sync", TRUE, NULL);
458 MH_ERROR(
"Error trying to create audio sink %s", asink_name);
461 const char *vsink_name = ::getenv(
"CORE_MEDIA_SERVICE_VIDEO_SINK_NAME");
463 if (vsink_name ==
nullptr) {
464 if (backend == lomiri::MediaHubService::AVBackend::Backend::hybris)
465 vsink_name = HYBRIS_SINK;
466 else if (backend == lomiri::MediaHubService::AVBackend::Backend::mir)
467 vsink_name = MIR_SINK;
471 video_sink_name = vsink_name;
472 video_sink = gst_element_factory_make (vsink_name,
"video-sink");
474 g_object_set (pipeline,
"video-sink", video_sink, NULL);
476 MH_ERROR(
"Error trying to create video sink %s", vsink_name);
482 if (not video_sink)
throw std::logic_error
484 "No video sink configured for the current pipeline"
487 setup_video_sink_for_buffer_streaming();
492 g_object_set (pipeline,
"volume", new_volume, NULL);
500 case media::Player::AudioStreamRole::alarm:
503 case media::Player::AudioStreamRole::alert:
506 case media::Player::AudioStreamRole::multimedia:
509 case media::Player::AudioStreamRole::phone:
520 if (g_strcmp0(orientation,
"rotate-0") == 0)
521 return media::Player::Orientation::rotate0;
522 else if (g_strcmp0(orientation,
"rotate-90") == 0)
523 return media::Player::Orientation::rotate90;
524 else if (g_strcmp0(orientation,
"rotate-180") == 0)
525 return media::Player::Orientation::rotate180;
526 else if (g_strcmp0(orientation,
"rotate-270") == 0)
527 return media::Player::Orientation::rotate270;
529 return media::Player::Orientation::rotate0;
535 const std::string role_str(
"props,media.role=" + get_audio_role_str(new_audio_role));
536 MH_INFO(
"Audio stream role: %s", role_str.c_str());
538 GstStructure *props = gst_structure_from_string (role_str.c_str(), NULL);
539 if (audio_sink !=
nullptr && props !=
nullptr)
541 g_object_set (audio_sink,
"stream-properties", props, NULL);
545 MH_WARNING(
"Couldn't set audio stream role - couldn't get audio_sink from pipeline");
548 gst_structure_free (props);
553 player_lifetime = lifetime;
559 gst_element_query_position (pipeline, GST_FORMAT_TIME, &pos);
563 if ((
static_cast<uint64_t
>(pos) < duration()) && is_seeking && pos == 0)
565 return previous_position;
570 previous_position =
static_cast<uint64_t
>(pos);
573 return static_cast<uint64_t
>(pos);
579 gst_element_query_duration (pipeline, GST_FORMAT_TIME, &dur);
582 return static_cast<uint64_t
>(dur);
588 bool do_pipeline_reset)
590 gchar *current_uri =
nullptr;
591 g_object_get(pipeline,
"current-uri", ¤t_uri, NULL);
596 if (current_uri and do_pipeline_reset)
599 QString tmp_uri{uri.toString(QUrl::FullyEncoded)};
600 g_object_set(pipeline,
"uri", qUtf8Printable(tmp_uri), NULL);
601 if (is_video_file(uri))
602 setMediaFileType(MEDIA_FILE_TYPE_VIDEO);
603 else if (is_audio_file(uri))
604 setMediaFileType(MEDIA_FILE_TYPE_AUDIO);
606 request_headers = headers;
608 if (!tmp_uri.isEmpty()) {
612 gst_element_set_state(pipeline, GST_STATE_PAUSED);
620 if (source == NULL || request_headers.isEmpty())
623 if (request_headers.find(
"Cookie") != request_headers.end()) {
624 if (g_object_class_find_property(G_OBJECT_GET_CLASS(source),
625 "cookies") != NULL) {
627 g_strsplit(qUtf8Printable(request_headers[
"Cookie"]),
";", 0);
628 g_object_set(source,
"cookies", cookies, NULL);
633 if (request_headers.find(
"User-Agent") != request_headers.end()) {
634 if (g_object_class_find_property(G_OBJECT_GET_CLASS(source),
635 "user-agent") != NULL) {
636 g_object_set(source,
"user-agent",
637 qUtf8Printable(request_headers[
"User-Agent"]), NULL);
642 if (request_headers.find(
"Authorization") != request_headers.end()) {
643 QString authString = request_headers[
"Authorization"];
645 if (authString.startsWith(
"Basic ")) {
646 authString = authString.mid(6);
649 QByteArray decodedAuth = QByteArray::fromBase64(authString.toUtf8());
650 int colonPos = decodedAuth.indexOf(
':');
652 const QByteArray user = decodedAuth.left(colonPos);
653 const QByteArray pass = decodedAuth.mid(colonPos + 1);
655 if (g_object_class_find_property(G_OBJECT_GET_CLASS(source),
"user-id") != NULL) {
656 g_object_set(source,
"user-id", user.constData(), NULL);
658 if (g_object_class_find_property(G_OBJECT_GET_CLASS(source),
"user-pw") != NULL) {
659 g_object_set(source,
"user-pw", pass.constData(), NULL);
667 int videoStreamCount = 0, audioStreamCount = 0;
668 g_object_get(pipeline,
"n-video", &videoStreamCount, NULL);
669 g_object_get(pipeline,
"n-audio", &audioStreamCount, NULL);
670 MH_DEBUG(
"streams changed: file has %d video streams and %d audio streams",
671 videoStreamCount, audioStreamCount);
673 if (videoStreamCount > 0)
674 setMediaFileType(MEDIA_FILE_TYPE_VIDEO);
675 else if (audioStreamCount > 0)
676 setMediaFileType(MEDIA_FILE_TYPE_AUDIO);
681 gchar* data =
nullptr;
682 g_object_get(pipeline,
"current-uri", &data,
nullptr);
684 QUrl result = QUrl::fromEncoded((data ==
nullptr ?
"" : data));
693 const auto ret = gst_element_set_state(pipeline, new_state);
695 MH_DEBUG(
"Requested state change.");
699 case GST_STATE_CHANGE_FAILURE:
700 result =
false;
break;
701 case GST_STATE_CHANGE_NO_PREROLL:
702 case GST_STATE_CHANGE_SUCCESS:
703 case GST_STATE_CHANGE_ASYNC:
704 result =
true;
break;
713 return gst_element_seek_simple(
716 (GstSeekFlags)(GST_SEEK_FLAG_FLUSH | GST_SEEK_FLAG_KEY_UNIT),
722 if (not video_sink || not is_supported_video_sink())
723 throw std::runtime_error
725 "Missing video sink or video sink does not support query of width and height."
729 int video_width = 0, video_height = 0;
732 GstIterator *iter = gst_element_iterate_pads(video_sink);
734 gst_iterator_next(iter, &item) == GST_ITERATOR_OK;
735 g_value_unset(&item))
737 GstPad *pad = GST_PAD(g_value_get_object(&item));
738 GstCaps *caps = gst_pad_get_current_caps(pad);
741 const GstStructure *s = gst_caps_get_structure(caps, 0);
742 gst_structure_get_int(s,
"width", &video_width);
743 gst_structure_get_int(s,
"height", &video_height);
744 MH_DEBUG(
"Video dimensions are %d x %d", video_width, video_height);
746 gst_caps_unref(caps);
749 gst_iterator_free(iter);
752 return QSize(video_width, video_height);
757 QMimeType mimeType = QMimeDatabase().mimeTypeForUrl(uri);
758 return mimeType.name();
766 QString content_type {file_info_from_uri(uri)};
767 if (content_type.isEmpty())
769 MH_WARNING(
"Failed to get actual track content type");
770 return QString(
"audio/video/");
773 MH_INFO(
"Found content type: %s", qUtf8Printable(content_type));
775 if (content_type ==
"video/3gpp") {
778 MH_INFO(
"Hack: remapping to audio/3gpp");
779 content_type =
"audio/3gpp";
789 if (get_file_content_type(uri).startsWith(
"audio/"))
791 MH_INFO(
"Found audio content");
803 if (get_file_content_type(uri).startsWith(
"video/"))
805 MH_INFO(
"Found video content");
828 if ((is_missing_audio_codec && audio_stream_id == -1) ||
829 (is_missing_video_codec && video_stream_id == -1) ||
830 (audio_stream_id == -1 && video_stream_id == -1))
836 bool gstreamer::Playbin::connect_to_consumer(
void)
838 static const char *local_socket =
"media-hub-server";
839 static const char *consumer_socket =
"media-consumer";
844 struct sockaddr_un local, remote;
846 if (sock_consumer != -1) {
848 close(sock_consumer);
851 if ((sock_consumer = socket(AF_UNIX, SOCK_DGRAM, 0)) == -1)
853 MH_ERROR(
"Cannot create socket: %s (%d)", strerror(errno), errno);
858 ostringstream local_ss;
859 local_ss << local_socket << key;
860 local.sun_family = AF_UNIX;
861 local.sun_path[0] =
'\0';
862 strcpy(local.sun_path + 1, local_ss.str().c_str());
863 len =
sizeof(local.sun_family) + local_ss.str().length() + 1;
864 if (bind(sock_consumer, (
struct sockaddr *) &local, len) == -1)
866 MH_ERROR(
"Cannot bind socket: %s (%d)", strerror(errno), errno);
867 close(sock_consumer);
873 ostringstream remote_ss;
874 remote_ss << consumer_socket << key;
875 remote.sun_family = AF_UNIX;
876 remote.sun_path[0] =
'\0';
877 strcpy(remote.sun_path + 1, remote_ss.str().c_str());
878 len =
sizeof(remote.sun_family) + remote_ss.str().length() + 1;
879 if (::connect(sock_consumer, (
struct sockaddr *) &remote, len) == -1)
881 MH_ERROR(
"Cannot connect to consumer: %s (%d)", strerror(errno), errno);
882 close(sock_consumer);
887 MH_DEBUG(
"Connected to buffer consumer socket");
892 void gstreamer::Playbin::send_buffer_data(
int fd,
void *data,
size_t len)
895 char buf[CMSG_SPACE(
sizeof fd)]{};
896 struct cmsghdr *cmsg;
897 struct iovec io = { .iov_base = data, .iov_len = len };
901 msg.msg_control = buf;
902 msg.msg_controllen =
sizeof buf;
904 cmsg = CMSG_FIRSTHDR(&msg);
905 cmsg->cmsg_level = SOL_SOCKET;
906 cmsg->cmsg_type = SCM_RIGHTS;
907 cmsg->cmsg_len = CMSG_LEN(
sizeof fd);
909 memmove(CMSG_DATA(cmsg), &fd,
sizeof fd);
911 msg.msg_controllen = cmsg->cmsg_len;
913 if (sendmsg(sock_consumer, &msg, 0) < 0)
914 MH_ERROR(
"Failed to send dma_buf fd to consumer: %s (%d)",
915 strerror(errno), errno);
918 void gstreamer::Playbin::send_frame_ready(
void)
920 const char ready =
'r';
922 if (send (sock_consumer, &ready,
sizeof ready, 0) == -1)
923 MH_ERROR(
"Error when sending frame ready flag to client: %s (%d)",
924 strerror(errno), errno);
929 if (fileType == m_fileType)
return;
930 m_fileType = fileType;
931 Q_EMIT mediaFileTypeChanged();