Music Hub  ..
A session-wide music playback service
playbin.cpp
Go to the documentation of this file.
1 /*
2  * Copyright © 2013-2015 Canonical Ltd.
3  * Copyright © 2022 UBports Foundation.
4  *
5  * Contact: Alberto Mardegan <mardy@users.sourceforge.net>
6  *
7  * This program is free software: you can redistribute it and/or modify it
8  * under the terms of the GNU Lesser General Public License version 3,
9  * as published by the Free Software Foundation.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public License
17  * along with this program. If not, see <http://www.gnu.org/licenses/>.
18  *
19  * Authored by: Thomas Voß <thomas.voss@canonical.com>
20  * Alfonso Sanchez-Beato <alfonso.sanchez-beato@canonical.com>
21  */
22 
23 #include "gstreamer/playbin.h"
24 
25 #include "gstreamer/engine.h"
26 #include "logging.h"
27 #include "video/socket_types.h"
28 #include "util/uri_check.h"
29 
30 #include <gst/pbutils/missing-plugins.h>
31 
32 #include <hybris/media/surface_texture_client_hybris.h>
33 #include <hybris/media/media_codec_layer.h>
34 
35 #include <QMimeDatabase>
36 #include <QMimeType>
37 #include <QSize>
38 
39 #include <sys/socket.h>
40 #include <sys/un.h>
41 
42 #include <sstream>
43 #include <utility>
44 #include <cstring>
45 
46 static const char *PULSE_SINK = "pulsesink";
47 static const char *HYBRIS_SINK = "hybrissink";
48 static const char *MIR_SINK = "mirsink";
49 
50 namespace media = lomiri::MediaHubService;
52 
53 void gstreamer::Playbin::setup_video_sink_for_buffer_streaming()
54 {
55  IGBPWrapperHybris igbp;
56  SurfaceTextureClientHybris stc;
57  GstContext *context;
58  GstStructure *structure;
59 
60  switch (backend) {
61  case lomiri::MediaHubService::AVBackend::Backend::hybris:
62  // Get the service-side BufferQueue (IGraphicBufferProducer) and
63  // associate with it the SurfaceTextureClientHybris instance.
64  igbp = decoding_service_get_igraphicbufferproducer();
65  stc = surface_texture_client_create_by_igbp(igbp);
66 
67  // Because mirsink is being loaded, we are definitely doing * hardware rendering.
68  surface_texture_client_set_hardware_rendering(stc, TRUE);
69 
70  context = gst_context_new("gst.mir.MirContext", TRUE);
71  structure = gst_context_writable_structure(context);
72  gst_structure_set(structure, "gst_mir_context", G_TYPE_POINTER, stc, NULL);
73 
74  /* Propagate context in pipeline (needed by amchybris and mirsink) */
75  gst_element_set_context(pipeline, context);
76  break;
77  case lomiri::MediaHubService::AVBackend::Backend::mir:
78  // Connect to buffer consumer socket
79  connect_to_consumer();
80  // Configure mirsink so it exports buffers (otherwise it would create
81  // its own window).
82  g_object_set (G_OBJECT (video_sink), "export-buffers", TRUE, nullptr);
83  break;
84  case lomiri::MediaHubService::AVBackend::Backend::none:
85  default:
86  throw lomiri::MediaHubService::Player::Errors::
87  OutOfProcessBufferStreamingNotSupported{};
88  }
89 }
90 
91 bool gstreamer::Playbin::is_supported_video_sink(void) const
92 {
93  if (video_sink_name == HYBRIS_SINK || video_sink_name == MIR_SINK)
94  return TRUE;
95 
96  return FALSE;
97 }
98 
99 // Uncomment to generate a dot file at the time that the pipeline
100 // goes to the PLAYING state. Make sure to export GST_DEBUG_DUMP_DOT_DIR
101 // before starting media-hub-server. To convert the dot file to something
102 // other image format, use: dot pipeline.dot -Tpng -o pipeline.png
103 //#define DEBUG_GST_PIPELINE
104 
106 {
107  static const std::string s{"playbin"};
108  return s;
109 }
110 
111 void gstreamer::Playbin::about_to_finish(GstElement*, gpointer user_data)
112 {
113  auto thiz = static_cast<Playbin*>(user_data);
114  Q_EMIT thiz->aboutToFinish();
115 }
116 
118  GstElement *source,
119  gpointer user_data)
120 {
121  if (user_data == nullptr)
122  return;
123 
124  static_cast<Playbin*>(user_data)->setup_source(source);
125 }
126 
127 void gstreamer::Playbin::streams_changed(GstElement *pipeline,
128  gpointer /*user_data*/)
129 {
130  /* We are possibly in a GStreamer working thread, so we notify the main
131  * thread of this event through a message in the bus */
132  gst_element_post_message(pipeline,
133  gst_message_new_application(GST_OBJECT(pipeline),
134  gst_structure_new_empty("streams-changed")));
135 }
136 
138  : pipeline(gst_element_factory_make("playbin", pipeline_name().c_str())),
139  bus{gst_element_get_bus(pipeline)},
140  m_fileType(MEDIA_FILE_TYPE_NONE),
141  video_sink(nullptr),
142  audio_sink(nullptr),
143  is_seeking(false),
144  previous_position(0),
145  player_lifetime(media::Player::Lifetime::normal),
146  about_to_finish_handler_id(0),
147  source_setup_handler_id(0),
148  is_missing_audio_codec(false),
149  is_missing_video_codec(false),
150  audio_stream_id(-1),
151  video_stream_id(-1),
152  current_new_state(GST_STATE_NULL),
153  key(key_in),
155  sock_consumer(-1)
156 {
157  if (!pipeline)
158  throw std::runtime_error("Could not create pipeline for playbin.");
159 
160  bus.onNewMessage([this](const Bus::Message &msg) {
161  on_new_message(msg);
162  });
163 
164  // Add audio and/or video sink elements depending on environment variables
165  // being set or not set
166  setup_pipeline_for_audio_video();
167 
168  about_to_finish_handler_id = g_signal_connect(
169  pipeline,
170  "about-to-finish",
171  G_CALLBACK(about_to_finish),
172  this
173  );
174 
175  source_setup_handler_id = g_signal_connect(
176  pipeline,
177  "source-setup",
178  G_CALLBACK(source_setup),
179  this
180  );
181 
182  m_audioChangedHandlerId = g_signal_connect(
183  pipeline, "audio-changed",
184  G_CALLBACK(streams_changed), this);
185 
186  m_videoChangedHandlerId = g_signal_connect(
187  pipeline, "video-changed",
188  G_CALLBACK(streams_changed), this);
189 }
190 
191 // Note that we might be accessing freed memory here, so activate DEBUG_REFS
192 // only for debugging
193 //#define DEBUG_REFS
194 #ifdef DEBUG_REFS
195 static void print_refs(const gstreamer::Playbin &pb, const char *func)
196 {
197  using namespace std;
198 
199  MH_DEBUG("%s", func);
200  if (pb.pipeline)
201  MH_DEBUG("pipeline: %d", (const void *) GST_OBJECT_REFCOUNT(pb.pipeline));
202  if (pb.video_sink)
203  MH_DEBUG("video_sink: %d", (const void *) GST_OBJECT_REFCOUNT(pb.video_sink));
204  if (pb.audio_sink)
205  MH_DEBUG("audio_sink: %d", (const void *) GST_OBJECT_REFCOUNT(pb.audio_sink));
206 }
207 #endif
208 
210 {
211 #ifdef DEBUG_REFS
212  print_refs(*this, "gstreamer::Playbin::~Playbin pipeline");
213 #endif
214 
215  g_signal_handler_disconnect(pipeline, about_to_finish_handler_id);
216  g_signal_handler_disconnect(pipeline, source_setup_handler_id);
217  g_signal_handler_disconnect(pipeline, m_audioChangedHandlerId);
218  g_signal_handler_disconnect(pipeline, m_videoChangedHandlerId);
219 
220  if (pipeline)
221  gst_object_unref(pipeline);
222 
223  if (sock_consumer != -1) {
224  close(sock_consumer);
225  sock_consumer = -1;
226  }
227 
228 #ifdef DEBUG_REFS
229  print_refs(*this, "gstreamer::Playbin::~Playbin pipeline");
230 #endif
231 }
232 
234 {
235  MH_INFO("Resetting pipeline");
236  // Tear down the current pipeline and get it
237  // in a state that is ready for the next client that connects to the
238  // service
239 
240  // Don't reset the pipeline if we want to resume
241  if (player_lifetime != media::Player::Lifetime::resumable) {
242  reset_pipeline();
243  }
244 }
245 
247 {
248  MH_TRACE("");
249  const auto ret = gst_element_set_state(pipeline, GST_STATE_NULL);
250  switch (ret)
251  {
252  case GST_STATE_CHANGE_FAILURE:
253  MH_WARNING("Failed to reset the pipeline state. Client reconnect may not function properly.");
254  break;
255  case GST_STATE_CHANGE_NO_PREROLL:
256  case GST_STATE_CHANGE_SUCCESS:
257  case GST_STATE_CHANGE_ASYNC:
258  break;
259  default:
260  MH_WARNING("Failed to reset the pipeline state. Client reconnect may not function properly.");
261  }
262  setMediaFileType(MEDIA_FILE_TYPE_NONE);
263  is_missing_audio_codec = false;
264  is_missing_video_codec = false;
265  audio_stream_id = -1;
266  video_stream_id = -1;
267  if (sock_consumer != -1) {
268  close(sock_consumer);
269  sock_consumer = -1;
270  }
271 }
272 
273 void gstreamer::Playbin::process_missing_plugin_message(GstMessage *message)
274 {
275  gchar *desc = gst_missing_plugin_message_get_description(message);
276  MH_WARNING("Missing plugin: %s", desc);
277  g_free(desc);
278 
279  const GstStructure *msg_data = gst_message_get_structure(message);
280  if (g_strcmp0("decoder", gst_structure_get_string(msg_data, "type")) != 0)
281  return;
282 
283  GstCaps *caps;
284  if (!gst_structure_get(msg_data, "detail", GST_TYPE_CAPS, &caps, NULL)) {
285  MH_ERROR("No detail");
286  return;
287  }
288 
289  GstStructure *caps_data = gst_caps_get_structure(caps, 0);
290  if (!caps_data) {
291  MH_ERROR("No caps data");
292  return;
293  }
294 
295  const gchar *mime = gst_structure_get_name(caps_data);
296  if (strstr(mime, "audio"))
297  is_missing_audio_codec = true;
298  else if (strstr(mime, "video"))
299  is_missing_video_codec = true;
300 
301  MH_ERROR("Missing decoder for %s", mime);
302 }
303 
306 {
307  if (state.new_state == GST_STATE_PAUSED ||
308  state.new_state == GST_STATE_PLAYING) {
309  // Get the video height/width from the video sink
310  try
311  {
312  const QSize new_dimensions = get_video_dimensions();
313  Q_EMIT videoDimensionChanged(new_dimensions);
314  }
315  catch (const std::exception& e)
316  {
317  MH_WARNING("Problem querying video dimensions: %s", e.what());
318  }
319  catch (...)
320  {
321  MH_WARNING("Problem querying video dimensions.");
322  }
323  }
324 }
325 
327 {
328  const GstStructure *msg_data = gst_message_get_structure(message);
329  const gchar *struct_name = gst_structure_get_name(msg_data);
330 
331  if (g_strcmp0("buffer-export-data", struct_name) == 0)
332  {
333  int fd;
335  if (!gst_structure_get(msg_data,
336  "fd", G_TYPE_INT, &fd,
337  "width", G_TYPE_INT, &meta.width,
338  "height", G_TYPE_INT, &meta.height,
339  "fourcc", G_TYPE_INT, &meta.fourcc,
340  "stride", G_TYPE_INT, &meta.stride,
341  "offset", G_TYPE_INT, &meta.offset,
342  NULL))
343  {
344  MH_ERROR("Bad buffer-export-data message: mirsink version mismatch?");
345  return;
346  }
347  MH_DEBUG("Exporting %dx%d buffer (fd %d)", meta.width, meta.height, fd);
348  send_buffer_data(fd, &meta, sizeof meta);
349  }
350  else if (g_strcmp0("frame-ready", struct_name) == 0)
351  {
352  send_frame_ready();
353  }
354  else if (g_strcmp0("streams-changed", struct_name) == 0)
355  {
356  updateMediaFileType();
357  }
358  else
359  {
360  MH_ERROR("Unknown GST_MESSAGE_ELEMENT with struct %s", struct_name);
361  }
362 }
363 
365 {
366  switch (message.type)
367  {
368  case GST_MESSAGE_ERROR:
369  Q_EMIT errorOccurred(message.detail.error_warning_info);
370  break;
371  case GST_MESSAGE_WARNING:
372  Q_EMIT warningOccurred(message.detail.error_warning_info);
373  break;
374  case GST_MESSAGE_INFO:
375  Q_EMIT infoOccurred(message.detail.error_warning_info);
376  break;
377  case GST_MESSAGE_STATE_CHANGED:
378  if (message.source == "playbin") {
379  g_object_get(G_OBJECT(pipeline), "current-audio", &audio_stream_id, NULL);
380  g_object_get(G_OBJECT(pipeline), "current-video", &video_stream_id, NULL);
381 #ifdef DEBUG_GST_PIPELINE
382  MH_DEBUG("Dumping pipeline dot file");
383  GST_DEBUG_BIN_TO_DOT_FILE((GstBin*)pipeline, GST_DEBUG_GRAPH_SHOW_ALL, "pipeline");
384 #endif
385 
386  // TODO: move here the stateChange() signal handling
387  // from gstreamer::Engine
388  } else if (message.source == "video-sink") {
389  processVideoSinkStateChanged(message.detail.state_changed);
390  }
391  Q_EMIT stateChanged(message.detail.state_changed, message.source);
392  break;
393  case GST_MESSAGE_APPLICATION:
394  case GST_MESSAGE_ELEMENT:
395  if (gst_is_missing_plugin_message(message.message))
396  process_missing_plugin_message(message.message);
397  else
398  process_message_element(message.message);
399  break;
400  case GST_MESSAGE_TAG:
401  {
402  gchar *orientation;
403  if (gst_tag_list_get_string(message.detail.tag.tag_list, "image-orientation", &orientation))
404  {
405  // If the image-orientation tag is in the GstTagList, signal the Engine
406  Q_EMIT orientationChanged(orientation_lut(orientation));
407  g_free (orientation);
408  }
409 
410  Q_EMIT tagAvailable(message.detail.tag);
411  }
412  break;
413  case GST_MESSAGE_ASYNC_DONE:
414  if (is_seeking)
415  {
416  // FIXME: Pass the actual playback time position to the signal call
417  Q_EMIT seekedTo(0);
418  is_seeking = false;
419  }
420  break;
421  case GST_MESSAGE_EOS:
422  Q_EMIT endOfStream();
423  break;
424  case GST_MESSAGE_BUFFERING:
425  Q_EMIT bufferingChanged(message.detail.buffering.percent);
426  break;
427  default:
428  break;
429  }
430 }
431 
433 {
434  return bus;
435 }
436 
438 {
439  gint flags;
440  g_object_get (pipeline, "flags", &flags, nullptr);
441  flags |= GST_PLAY_FLAG_AUDIO;
442  flags |= GST_PLAY_FLAG_VIDEO;
443  flags &= ~GST_PLAY_FLAG_TEXT;
444  g_object_set (pipeline, "flags", flags, nullptr);
445 
446  const char *asink_name = ::getenv("CORE_MEDIA_SERVICE_AUDIO_SINK_NAME");
447 
448  if (asink_name == nullptr)
449  asink_name = PULSE_SINK;
450 
451  audio_sink = gst_element_factory_make (asink_name, "audio-sink");
452  if (audio_sink) {
453  g_object_set (pipeline, "audio-sink", audio_sink, NULL);
454  if (strcmp(asink_name, "fakesink") == 0) {
455  g_object_set(audio_sink, "sync", TRUE, NULL);
456  }
457  } else {
458  MH_ERROR("Error trying to create audio sink %s", asink_name);
459  }
460 
461  const char *vsink_name = ::getenv("CORE_MEDIA_SERVICE_VIDEO_SINK_NAME");
462 
463  if (vsink_name == nullptr) {
464  if (backend == lomiri::MediaHubService::AVBackend::Backend::hybris)
465  vsink_name = HYBRIS_SINK;
466  else if (backend == lomiri::MediaHubService::AVBackend::Backend::mir)
467  vsink_name = MIR_SINK;
468  }
469 
470  if (vsink_name) {
471  video_sink_name = vsink_name;
472  video_sink = gst_element_factory_make (vsink_name, "video-sink");
473  if (video_sink)
474  g_object_set (pipeline, "video-sink", video_sink, NULL);
475  else
476  MH_ERROR("Error trying to create video sink %s", vsink_name);
477  }
478 }
479 
481 {
482  if (not video_sink) throw std::logic_error
483  {
484  "No video sink configured for the current pipeline"
485  };
486 
487  setup_video_sink_for_buffer_streaming();
488 }
489 
490 void gstreamer::Playbin::set_volume(double new_volume)
491 {
492  g_object_set (pipeline, "volume", new_volume, NULL);
493 }
494 
497 {
498  switch (audio_role)
499  {
500  case media::Player::AudioStreamRole::alarm:
501  return "alarm";
502  break;
503  case media::Player::AudioStreamRole::alert:
504  return "alert";
505  break;
506  case media::Player::AudioStreamRole::multimedia:
507  return "multimedia";
508  break;
509  case media::Player::AudioStreamRole::phone:
510  return "phone";
511  break;
512  default:
513  return "multimedia";
514  break;
515  }
516 }
517 
519 {
520  if (g_strcmp0(orientation, "rotate-0") == 0)
521  return media::Player::Orientation::rotate0;
522  else if (g_strcmp0(orientation, "rotate-90") == 0)
523  return media::Player::Orientation::rotate90;
524  else if (g_strcmp0(orientation, "rotate-180") == 0)
525  return media::Player::Orientation::rotate180;
526  else if (g_strcmp0(orientation, "rotate-270") == 0)
527  return media::Player::Orientation::rotate270;
528  else
529  return media::Player::Orientation::rotate0;
530 }
531 
534 {
535  const std::string role_str("props,media.role=" + get_audio_role_str(new_audio_role));
536  MH_INFO("Audio stream role: %s", role_str.c_str());
537 
538  GstStructure *props = gst_structure_from_string (role_str.c_str(), NULL);
539  if (audio_sink != nullptr && props != nullptr)
540  {
541  g_object_set (audio_sink, "stream-properties", props, NULL);
542  }
543  else
544  {
545  MH_WARNING("Couldn't set audio stream role - couldn't get audio_sink from pipeline");
546  }
547 
548  gst_structure_free (props);
549 }
550 
552 {
553  player_lifetime = lifetime;
554 }
555 
557 {
558  int64_t pos = 0;
559  gst_element_query_position (pipeline, GST_FORMAT_TIME, &pos);
560 
561  // This prevents a 0 position from being reported to the app which happens while seeking.
562  // This is covering over a GStreamer issue
563  if ((static_cast<uint64_t>(pos) < duration()) && is_seeking && pos == 0)
564  {
565  return previous_position;
566  }
567 
568  // Save the current position to use just in case it's needed the next time position is
569  // requested
570  previous_position = static_cast<uint64_t>(pos);
571 
572  // FIXME: this should be int64_t, but dbus-cpp doesn't seem to handle it correctly
573  return static_cast<uint64_t>(pos);
574 }
575 
577 {
578  int64_t dur = 0;
579  gst_element_query_duration (pipeline, GST_FORMAT_TIME, &dur);
580 
581  // FIXME: this should be int64_t, but dbus-cpp doesn't seem to handle it correctly
582  return static_cast<uint64_t>(dur);
583 }
584 
586  const QUrl &uri,
587  const media::Player::HeadersType &headers,
588  bool do_pipeline_reset)
589 {
590  gchar *current_uri = nullptr;
591  g_object_get(pipeline, "current-uri", &current_uri, NULL);
592 
593  // Checking for a current_uri being set and not resetting the pipeline
594  // if there isn't a current_uri causes the first play to start playback
595  // sooner since reset_pipeline won't be called
596  if (current_uri and do_pipeline_reset)
597  reset_pipeline();
598 
599  QString tmp_uri{uri.toString(QUrl::FullyEncoded)};
600  g_object_set(pipeline, "uri", qUtf8Printable(tmp_uri), NULL);
601  if (is_video_file(uri))
602  setMediaFileType(MEDIA_FILE_TYPE_VIDEO);
603  else if (is_audio_file(uri))
604  setMediaFileType(MEDIA_FILE_TYPE_AUDIO);
605 
606  request_headers = headers;
607 
608  if (!tmp_uri.isEmpty()) {
609  /* Setting the pipeline to "paused" to let GStreamer inspect the media
610  * and report the number of audio and video streams
611  */
612  gst_element_set_state(pipeline, GST_STATE_PAUSED);
613  }
614 
615  g_free(current_uri);
616 }
617 
618 void gstreamer::Playbin::setup_source(GstElement *source)
619 {
620  if (source == NULL || request_headers.isEmpty())
621  return;
622 
623  if (request_headers.find("Cookie") != request_headers.end()) {
624  if (g_object_class_find_property(G_OBJECT_GET_CLASS(source),
625  "cookies") != NULL) {
626  gchar ** cookies =
627  g_strsplit(qUtf8Printable(request_headers["Cookie"]), ";", 0);
628  g_object_set(source, "cookies", cookies, NULL);
629  g_strfreev(cookies);
630  }
631  }
632 
633  if (request_headers.find("User-Agent") != request_headers.end()) {
634  if (g_object_class_find_property(G_OBJECT_GET_CLASS(source),
635  "user-agent") != NULL) {
636  g_object_set(source, "user-agent",
637  qUtf8Printable(request_headers["User-Agent"]), NULL);
638  }
639  }
640 
641  // Re-interpret "Authorization" header into user and password properties
642  if (request_headers.find("Authorization") != request_headers.end()) {
643  QString authString = request_headers["Authorization"];
644 
645  if (authString.startsWith("Basic ")) {
646  authString = authString.mid(6);
647  }
648 
649  QByteArray decodedAuth = QByteArray::fromBase64(authString.toUtf8());
650  int colonPos = decodedAuth.indexOf(':');
651  if (colonPos >= 0) {
652  const QByteArray user = decodedAuth.left(colonPos);
653  const QByteArray pass = decodedAuth.mid(colonPos + 1);
654 
655  if (g_object_class_find_property(G_OBJECT_GET_CLASS(source), "user-id") != NULL) {
656  g_object_set(source, "user-id", user.constData(), NULL);
657  }
658  if (g_object_class_find_property(G_OBJECT_GET_CLASS(source), "user-pw") != NULL) {
659  g_object_set(source, "user-pw", pass.constData(), NULL);
660  }
661  }
662  }
663 }
664 
666 {
667  int videoStreamCount = 0, audioStreamCount = 0;
668  g_object_get(pipeline, "n-video", &videoStreamCount, NULL);
669  g_object_get(pipeline, "n-audio", &audioStreamCount, NULL);
670  MH_DEBUG("streams changed: file has %d video streams and %d audio streams",
671  videoStreamCount, audioStreamCount);
672 
673  if (videoStreamCount > 0)
674  setMediaFileType(MEDIA_FILE_TYPE_VIDEO);
675  else if (audioStreamCount > 0)
676  setMediaFileType(MEDIA_FILE_TYPE_AUDIO);
677 }
678 
680 {
681  gchar* data = nullptr;
682  g_object_get(pipeline, "current-uri", &data, nullptr);
683 
684  QUrl result = QUrl::fromEncoded((data == nullptr ? "" : data));
685  g_free(data);
686 
687  return result;
688 }
689 
690 bool gstreamer::Playbin::set_state(GstState new_state)
691 {
692  bool result = false;
693  const auto ret = gst_element_set_state(pipeline, new_state);
694 
695  MH_DEBUG("Requested state change.");
696 
697  switch (ret)
698  {
699  case GST_STATE_CHANGE_FAILURE:
700  result = false; break;
701  case GST_STATE_CHANGE_NO_PREROLL:
702  case GST_STATE_CHANGE_SUCCESS:
703  case GST_STATE_CHANGE_ASYNC:
704  result = true; break;
705  }
706 
707  return result;
708 }
709 
710 bool gstreamer::Playbin::seek(const std::chrono::microseconds& ms)
711 {
712  is_seeking = true;
713  return gst_element_seek_simple(
714  pipeline,
715  GST_FORMAT_TIME,
716  (GstSeekFlags)(GST_SEEK_FLAG_FLUSH | GST_SEEK_FLAG_KEY_UNIT),
717  ms.count() * 1000);
718 }
719 
721 {
722  if (not video_sink || not is_supported_video_sink())
723  throw std::runtime_error
724  {
725  "Missing video sink or video sink does not support query of width and height."
726  };
727 
728  // Initialize to default value prior to querying actual values from the sink.
729  int video_width = 0, video_height = 0;
730 
731  // There should be only one pad actually
732  GstIterator *iter = gst_element_iterate_pads(video_sink);
733  for (GValue item{};
734  gst_iterator_next(iter, &item) == GST_ITERATOR_OK;
735  g_value_unset(&item))
736  {
737  GstPad *pad = GST_PAD(g_value_get_object(&item));
738  GstCaps *caps = gst_pad_get_current_caps(pad);
739 
740  if (caps) {
741  const GstStructure *s = gst_caps_get_structure(caps, 0);
742  gst_structure_get_int(s, "width", &video_width);
743  gst_structure_get_int(s, "height", &video_height);
744  MH_DEBUG("Video dimensions are %d x %d", video_width, video_height);
745 
746  gst_caps_unref(caps);
747  }
748  }
749  gst_iterator_free(iter);
750 
751  // TODO(tvoss): We should probably check here if width and height are valid.
752  return QSize(video_width, video_height);
753 }
754 
755 QString gstreamer::Playbin::file_info_from_uri(const QUrl &uri) const
756 {
757  QMimeType mimeType = QMimeDatabase().mimeTypeForUrl(uri);
758  return mimeType.name();
759 }
760 
761 QString gstreamer::Playbin::get_file_content_type(const QUrl &uri) const
762 {
763  if (uri.isEmpty())
764  return QString();
765 
766  QString content_type {file_info_from_uri(uri)};
767  if (content_type.isEmpty())
768  {
769  MH_WARNING("Failed to get actual track content type");
770  return QString("audio/video/");
771  }
772 
773  MH_INFO("Found content type: %s", qUtf8Printable(content_type));
774 
775  if (content_type == "video/3gpp") {
776  /* ogg files recorded by messaging app as detected as video/3gpp,
777  * which causes the playback to fail. Hack around it: */
778  MH_INFO("Hack: remapping to audio/3gpp");
779  content_type = "audio/3gpp";
780  }
781  return content_type;
782 }
783 
784 bool gstreamer::Playbin::is_audio_file(const QUrl &uri) const
785 {
786  if (uri.isEmpty())
787  return false;
788 
789  if (get_file_content_type(uri).startsWith("audio/"))
790  {
791  MH_INFO("Found audio content");
792  return true;
793  }
794 
795  return false;
796 }
797 
798 bool gstreamer::Playbin::is_video_file(const QUrl &uri) const
799 {
800  if (uri.isEmpty())
801  return false;
802 
803  if (get_file_content_type(uri).startsWith("video/"))
804  {
805  MH_INFO("Found video content");
806  return true;
807  }
808 
809  return false;
810 }
811 
813 {
814  return m_fileType;
815 }
816 
818 {
819  /*
820  * We do not consider that we can play the video when
821  * 1. No audio stream selected due to missing decoder
822  * 2. No video stream selected due to missing decoder
823  * 3. No stream selected at all
824  * Note that if there are several, say, audio streams, we will play the file
825  * provided that we can decode just one of them, even if there are missing
826  * audio codecs. We will also play files with only one type of stream.
827  */
828  if ((is_missing_audio_codec && audio_stream_id == -1) ||
829  (is_missing_video_codec && video_stream_id == -1) ||
830  (audio_stream_id == -1 && video_stream_id == -1))
831  return false;
832  else
833  return true;
834 }
835 
836 bool gstreamer::Playbin::connect_to_consumer(void)
837 {
838  static const char *local_socket = "media-hub-server";
839  static const char *consumer_socket = "media-consumer";
840 
841  using namespace std;
842 
843  int len;
844  struct sockaddr_un local, remote;
845 
846  if (sock_consumer != -1) {
847  MH_DEBUG("Resetting socket");
848  close(sock_consumer);
849  }
850 
851  if ((sock_consumer = socket(AF_UNIX, SOCK_DGRAM, 0)) == -1)
852  {
853  MH_ERROR("Cannot create socket: %s (%d)", strerror(errno), errno);
854  return false;
855  }
856 
857  // Bind client to local -abstract- socket (media-hub-server<session>)
858  ostringstream local_ss;
859  local_ss << local_socket << key;
860  local.sun_family = AF_UNIX;
861  local.sun_path[0] = '\0';
862  strcpy(local.sun_path + 1, local_ss.str().c_str());
863  len = sizeof(local.sun_family) + local_ss.str().length() + 1;
864  if (bind(sock_consumer, (struct sockaddr *) &local, len) == -1)
865  {
866  MH_ERROR("Cannot bind socket: %s (%d)", strerror(errno), errno);
867  close(sock_consumer);
868  sock_consumer = -1;
869  return false;
870  }
871 
872  // Connect to buffer consumer (media-consumer<session>)
873  ostringstream remote_ss;
874  remote_ss << consumer_socket << key;
875  remote.sun_family = AF_UNIX;
876  remote.sun_path[0] = '\0';
877  strcpy(remote.sun_path + 1, remote_ss.str().c_str());
878  len = sizeof(remote.sun_family) + remote_ss.str().length() + 1;
879  if (::connect(sock_consumer, (struct sockaddr *) &remote, len) == -1)
880  {
881  MH_ERROR("Cannot connect to consumer: %s (%d)", strerror(errno), errno);
882  close(sock_consumer);
883  sock_consumer = -1;
884  return false;
885  }
886 
887  MH_DEBUG("Connected to buffer consumer socket");
888 
889  return true;
890 }
891 
892 void gstreamer::Playbin::send_buffer_data(int fd, void *data, size_t len)
893 {
894  struct msghdr msg{};
895  char buf[CMSG_SPACE(sizeof fd)]{};
896  struct cmsghdr *cmsg;
897  struct iovec io = { .iov_base = data, .iov_len = len };
898 
899  msg.msg_iov = &io;
900  msg.msg_iovlen = 1;
901  msg.msg_control = buf;
902  msg.msg_controllen = sizeof buf;
903 
904  cmsg = CMSG_FIRSTHDR(&msg);
905  cmsg->cmsg_level = SOL_SOCKET;
906  cmsg->cmsg_type = SCM_RIGHTS;
907  cmsg->cmsg_len = CMSG_LEN(sizeof fd);
908 
909  memmove(CMSG_DATA(cmsg), &fd, sizeof fd);
910 
911  msg.msg_controllen = cmsg->cmsg_len;
912 
913  if (sendmsg(sock_consumer, &msg, 0) < 0)
914  MH_ERROR("Failed to send dma_buf fd to consumer: %s (%d)",
915  strerror(errno), errno);
916 }
917 
918 void gstreamer::Playbin::send_frame_ready(void)
919 {
920  const char ready = 'r';
921 
922  if (send (sock_consumer, &ready, sizeof ready, 0) == -1)
923  MH_ERROR("Error when sending frame ready flag to client: %s (%d)",
924  strerror(errno), errno);
925 }
926 
928 {
929  if (fileType == m_fileType) return;
930  m_fileType = fileType;
931  Q_EMIT mediaFileTypeChanged();
932 }
socket_types.h
gstreamer::Playbin::set_uri
void set_uri(const QUrl &uri, const lomiri::MediaHubService::Player::HeadersType &headers, bool do_pipeline_reset=true)
Definition: playbin.cpp:585
gstreamer::Bus::Message::Detail::percent
gint percent
Definition: bus.h:203
gstreamer::Playbin
Definition: playbin.h:46
gstreamer::Playbin::setup_pipeline_for_audio_video
void setup_pipeline_for_audio_video()
Definition: playbin.cpp:437
gstreamer::Bus::Message
Definition: bus.h:39
lomiri::MediaHubService::video::BufferMetadata::fourcc
int fourcc
Definition: socket_types.h:35
lomiri::MediaHubService::video::BufferMetadata::offset
int offset
Definition: socket_types.h:37
gstreamer::Bus::Message::Detail::StateChanged::new_state
GstState new_state
Definition: bus.h:215
gstreamer::Playbin::audio_sink
GstElement * audio_sink
Definition: playbin.h:131
gstreamer::Playbin::get_audio_role_str
static std::string get_audio_role_str(lomiri::MediaHubService::Player::AudioStreamRole audio_role)
Definition: playbin.cpp:496
gstreamer::Playbin::set_audio_stream_role
void set_audio_stream_role(lomiri::MediaHubService::Player::AudioStreamRole new_audio_role)
Definition: playbin.cpp:533
gstreamer::Bus::Message::Detail::buffering
struct gstreamer::Bus::Message::Detail::@0 buffering
gstreamer::Playbin::position
uint64_t position() const
Definition: playbin.cpp:556
gstreamer::Playbin::get_video_dimensions
QSize get_video_dimensions() const
Definition: playbin.cpp:720
gstreamer::Bus
Definition: bus.h:36
gstreamer::Playbin::message_bus
gstreamer::Bus & message_bus()
Definition: playbin.cpp:432
gstreamer::Playbin::reset
void reset()
Definition: playbin.cpp:233
gstreamer::Playbin::duration
uint64_t duration() const
Definition: playbin.cpp:576
lomiri::MediaHubService::Player::HeadersType
QMap< QString, QString > HeadersType
Definition: player.h:57
gstreamer::Playbin::aboutToFinish
void aboutToFinish()
gstreamer::Playbin::updateMediaFileType
void updateMediaFileType()
Definition: playbin.cpp:665
gstreamer::Playbin::processVideoSinkStateChanged
void processVideoSinkStateChanged(const Bus::Message::Detail::StateChanged &state)
Definition: playbin.cpp:304
gstreamer::Playbin::set_volume
void set_volume(double new_volume)
Definition: playbin.cpp:490
gstreamer::Playbin::can_play_streams
bool can_play_streams() const
Definition: playbin.cpp:817
gstreamer::Playbin::get_file_content_type
QString get_file_content_type(const QUrl &uri) const
Definition: playbin.cpp:761
lomiri::MediaHubService::Player::AudioStreamRole
AudioStreamRole
Definition: player.h:113
gstreamer::Playbin::on_new_message
void on_new_message(const Bus::Message &message)
Definition: playbin.cpp:364
gstreamer::Playbin::set_state
bool set_state(GstState new_state)
Definition: playbin.cpp:690
gstreamer::Playbin::is_audio_file
bool is_audio_file(const QUrl &uri) const
Definition: playbin.cpp:784
lomiri::MediaHubService::Player::PlayerKey
uint32_t PlayerKey
Definition: player.h:55
lomiri::MediaHubService::AVBackend::get_backend_type
static Backend get_backend_type()
Returns the type of audio/video decoding/encoding backend being used.
Definition: backend.cpp:28
gstreamer::Playbin::orientation_lut
lomiri::MediaHubService::Player::Orientation orientation_lut(const gchar *orientation)
Definition: playbin.cpp:518
gstreamer::Bus::Message::Detail::error_warning_info
struct gstreamer::Bus::Message::Detail::ErrorWarningInfo error_warning_info
gstreamer::Playbin::setMediaFileType
void setMediaFileType(MediaFileType fileType)
Definition: playbin.cpp:927
MH_ERROR
#define MH_ERROR(...)
Definition: logging.h:41
gstreamer::Playbin::mediaFileType
MediaFileType mediaFileType() const
Definition: playbin.cpp:812
gstreamer::Bus::Message::detail
union gstreamer::Bus::Message::Detail detail
gstreamer::Playbin::pipeline
GstElement * pipeline
Definition: playbin.h:127
gstreamer::Playbin::setup_source
void setup_source(GstElement *source)
Definition: playbin.cpp:618
gstreamer::Playbin::source_setup
static void source_setup(GstElement *, GstElement *source, gpointer user_data)
Definition: playbin.cpp:117
gstreamer::Playbin::reset_pipeline
void reset_pipeline()
Definition: playbin.cpp:246
MH_TRACE
#define MH_TRACE(...)
Definition: logging.h:37
gstreamer::Bus::Message::Detail::tag
struct gstreamer::Bus::Message::Detail::Tag tag
lomiri::MediaHubService
Definition: context.h:28
lomiri::MediaHubService::video::BufferMetadata::width
int width
Definition: socket_types.h:33
gstreamer::Playbin::uri
QUrl uri() const
Definition: playbin.cpp:679
gstreamer::Bus::Message::Detail::Tag::tag_list
GstTagList * tag_list
Definition: bus.h:199
gstreamer::Playbin::is_video_file
bool is_video_file(const QUrl &uri) const
Definition: playbin.cpp:798
lomiri::MediaHubService::video::BufferMetadata
Definition: socket_types.h:31
gstreamer::Playbin::Playbin
Playbin(const lomiri::MediaHubService::Player::PlayerKey key)
Definition: playbin.cpp:137
gstreamer::Playbin::pipeline_name
static const std::string & pipeline_name()
Definition: playbin.cpp:105
gstreamer::Bus::Message::source
QByteArray source
Definition: bus.h:183
lomiri::MediaHubService::Player::Lifetime
Lifetime
Definition: player.h:129
gstreamer::Playbin::seek
bool seek(const std::chrono::microseconds &ms)
Definition: playbin.cpp:710
gstreamer::Playbin::file_info_from_uri
QString file_info_from_uri(const QUrl &uri) const
Definition: playbin.cpp:755
gstreamer::Playbin::streams_changed
static void streams_changed(GstElement *, gpointer user_data)
Definition: playbin.cpp:127
gstreamer::Playbin::create_video_sink
void create_video_sink(uint32_t texture_id)
Definition: playbin.cpp:480
gstreamer::Playbin::process_message_element
void process_message_element(GstMessage *message)
Definition: playbin.cpp:326
MH_DEBUG
#define MH_DEBUG(...)
Definition: logging.h:38
gstreamer::Bus::Message::type
GstMessageType type
Definition: bus.h:182
gstreamer::Playbin::video_sink
GstElement * video_sink
Definition: playbin.h:130
gstreamer::Playbin::MediaFileType
MediaFileType
Definition: playbin.h:58
lomiri::MediaHubService::video::BufferMetadata::stride
int stride
Definition: socket_types.h:36
MH_INFO
#define MH_INFO(...)
Definition: logging.h:39
lomiri::MediaHubService::video
Definition: socket_types.h:28
gstreamer::Bus::Message::message
GstMessage * message
Definition: bus.h:181
uri_check.h
gstreamer::Playbin::about_to_finish
static void about_to_finish(GstElement *, gpointer user_data)
Definition: playbin.cpp:111
gstreamer::Bus::Message::Detail::state_changed
struct gstreamer::Bus::Message::Detail::StateChanged state_changed
engine.h
gstreamer::Bus::Message::Detail::StateChanged
Definition: bus.h:212
gstreamer::Playbin::~Playbin
~Playbin()
Definition: playbin.cpp:209
MH_WARNING
#define MH_WARNING(...)
Definition: logging.h:40
gstreamer::Playbin::set_lifetime
void set_lifetime(lomiri::MediaHubService::Player::Lifetime)
Definition: playbin.cpp:551
playbin.h
lomiri::MediaHubService::Player::Orientation
Orientation
Definition: player.h:121
lomiri::MediaHubService::video::BufferMetadata::height
int height
Definition: socket_types.h:34