Music Hub  ..
A session-wide music playback service
egl_video_sink.cpp
Go to the documentation of this file.
1 /*
2  * Copyright © 2021-2022 UBports Foundation.
3  *
4  * Contact: Alberto Mardegan <mardy@users.sourceforge.net>
5  *
6  * This program is free software: you can redistribute it and/or modify it
7  * under the terms of the GNU Lesser General Public License version 3,
8  * as published by the Free Software Foundation.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13  * GNU Lesser General Public License for more details.
14  *
15  * You should have received a copy of the GNU Lesser General Public License
16  * along with this program. If not, see <http://www.gnu.org/licenses/>.
17  */
18 
19 #include "egl_video_sink.h"
20 
21 #include "logging.h"
22 #include "socket_types.h"
23 
24 #include <EGL/egl.h>
25 #include <EGL/eglext.h>
26 #include <GLES2/gl2.h>
27 #include <GLES2/gl2ext.h>
28 
29 #include <sys/types.h>
30 #include <sys/socket.h>
31 #include <sys/un.h>
32 
33 #include <sstream>
34 #include <thread>
35 #include <future>
36 #include <cstring>
37 #include <unistd.h>
38 
39 using namespace lomiri::MediaHub;
40 using namespace std;
41 
42 namespace lomiri {
43 namespace MediaHub {
44 
46 {
47  friend class EglVideoSink;
48 
49  static bool receive_buff(int socket, BufferData *data)
50  {
51  struct msghdr msg{};
52  struct iovec io = { .iov_base = &data->meta,
53  .iov_len = sizeof data->meta };
54  char c_buffer[256];
55  ssize_t res;
56 
57  msg.msg_iov = &io;
58  msg.msg_iovlen = 1;
59 
60  msg.msg_control = c_buffer;
61  msg.msg_controllen = sizeof c_buffer;
62 
63  if ((res = recvmsg(socket, &msg, 0)) == -1) {
64  MH_ERROR("Failed to receive message");
65  return false;
66  } else if (res == 0) {
67  MH_ERROR("Socket shutdown while receiving buffer data");
68  return false;
69  }
70 
71  struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg);
72 
73  memmove(&data->fd, CMSG_DATA(cmsg), sizeof data->fd);
74 
75  MH_DEBUG("Extracted fd %d", data->fd);
76  MH_DEBUG("width %d", data->meta.width);
77  MH_DEBUG("height %d", data->meta.height);
78  MH_DEBUG("fourcc 0x%X", data->meta.fourcc);
79  MH_DEBUG("stride %d", data->meta.stride);
80  MH_DEBUG("offset %d", data->meta.offset);
81 
82  return true;
83  }
84 
85  static void read_sock_events(PlayerKey key,
86  int sock_fd,
87  promise<BufferData>& prom_buff,
88  EglVideoSink *q)
89  {
90  static const char *consumer_socket = "media-consumer";
91 
92  struct sockaddr_un local;
93  int len;
94  BufferData buff_data;
95 
96  if (sock_fd == -1) {
97  MH_ERROR("Cannot create buffer consumer socket: %s (%d)",
98  strerror(errno), errno);
99  return;
100  }
101 
102  ostringstream sock_name_ss;
103  sock_name_ss << consumer_socket << key;
104  local.sun_family = AF_UNIX;
105  local.sun_path[0] = '\0';
106  strcpy(local.sun_path + 1, sock_name_ss.str().c_str());
107  len = sizeof(local.sun_family) + sock_name_ss.str().length() + 1;
108  if (bind(sock_fd, (struct sockaddr *) &local, len) == -1) {
109  MH_ERROR("Cannot bind consumer socket: %s (%d)",
110  strerror(errno), errno);
111  return;
112  }
113 
114  // Wait for buffer descriptions, pass them to rendering thread
115  if (!receive_buff(sock_fd, &buff_data))
116  return;
117 
118  prom_buff.set_value(buff_data);
119 
120  // Now signal frame syncs
121  while(true) {
122  ssize_t res;
123  char c;
124 
125  res = recv(sock_fd, &c, sizeof c, 0);
126  if (res == -1) {
127  MH_ERROR("while waiting sync: %s (%d)",
128  strerror(errno), errno);
129  return;
130  } else if (res == 0) {
131  MH_DEBUG("Socket shutdown");
132  return;
133  }
134 
135  Q_EMIT q->frameAvailable();
136  }
137  }
138 
139  bool find_extension(const string& extensions, const string& ext)
140  {
141  size_t len_all = extensions.length();
142  size_t len = ext.length();
143  size_t pos = 0;
144 
145  while ((pos = extensions.find(ext, pos)) != string::npos) {
146  if (pos + len == len_all || extensions[pos + len] == ' ')
147  return true;
148 
149  pos = pos + len;
150  }
151 
152  return false;
153  }
154 
155 public:
156  EglVideoSinkPrivate(uint32_t gl_texture, PlayerKey key,
157  EglVideoSink *q):
158  gl_texture{gl_texture},
159  prom_buff{},
160  fut_buff{prom_buff.get_future()},
161  sock_fd{socket(AF_UNIX, SOCK_DGRAM, 0)},
162  sock_thread{read_sock_events, key, sock_fd,
163  ref(prom_buff), q},
164  egl_image{EGL_NO_IMAGE_KHR},
165  buf_fd{-1}
166  {
167  const char *extensions;
168  const char *egl_needed[] = {"EGL_KHR_image_base",
169  "EGL_EXT_image_dma_buf_import"};
170  EGLDisplay egl_display = eglGetCurrentDisplay();
171  size_t i;
172 
173  // Retrieve EGL extensions from current display, then make sure the ones
174  // we need are present.
175  extensions = eglQueryString (egl_display, EGL_EXTENSIONS);
176  if (!extensions)
177  throw runtime_error {"Error querying EGL extensions"};
178 
179  for (i = 0; i < sizeof(egl_needed)/sizeof(egl_needed[0]); ++i) {
180  if (!find_extension(extensions, egl_needed[i])) {
181  MH_DEBUG("%s not supported", egl_needed[i]);
182  //ostringstream oss;
183  //oss << egl_needed[i] << " not supported";
184  // TODO: The returned extensions do not really reflect what is
185  // supported by the system, and do not include the ones we need.
186  // It is probably related to how qt initializes EGL, because
187  // mirsink does not show this problem. So we need to
188  // check why extensions is different from es2_info output.
189  //throw runtime_error {oss.str().c_str()};
190  }
191  }
192 
193  // TODO this returns a NULL pointer, probably same issue as with eglQueryString
194  // extensions = reinterpret_cast<const char *>(glGetString(GL_EXTENSIONS));
195  // if (!extensions)
196  // throw runtime_error {"Error querying OpenGL ES extensions"};
197 
198  // if (!find_extension(extensions, "GL_OES_EGL_image_external"))
199  // throw runtime_error {"GL_OES_EGL_image_external is not supported"};
200 
201  // Dynamically load functions from extensions (they are not
202  // automatically exported by EGL library).
203  _eglCreateImageKHR = (PFNEGLCREATEIMAGEKHRPROC)
204  eglGetProcAddress("eglCreateImageKHR");
205  _eglDestroyImageKHR = (PFNEGLDESTROYIMAGEKHRPROC)
206  eglGetProcAddress("eglDestroyImageKHR");
207  _glEGLImageTargetTexture2DOES = (PFNGLEGLIMAGETARGETTEXTURE2DOESPROC)
208  eglGetProcAddress("glEGLImageTargetTexture2DOES");
209 
210  if (_eglCreateImageKHR == nullptr || _eglDestroyImageKHR == nullptr ||
211  _glEGLImageTargetTexture2DOES == nullptr)
212  throw runtime_error {"Error when loading extensions"};
213  }
214 
216  {
217  if (sock_fd != -1) {
218  shutdown(sock_fd, SHUT_RDWR);
219  sock_thread.join();
220  close(sock_fd);
221  }
222 
223  if (buf_fd != -1)
224  close(buf_fd);
225 
226  if (egl_image != EGL_NO_IMAGE_KHR)
227  _eglDestroyImageKHR(eglGetCurrentDisplay(), egl_image);
228  }
229 
230  // This imports dma_buf buffers by using the EGL_EXT_image_dma_buf_import
231  // extension. The buffers have been previously exported in mirsink, using
232  // EGL_MESA_image_dma_buf_export extension. After that, we bind the buffer
233  // to the app texture by using GL_OES_EGL_image_external extension.
234  bool import_buffer(const BufferData *buf_data)
235  {
236  GLenum err;
237  EGLDisplay egl_display = eglGetCurrentDisplay();
238  EGLint image_attrs[] = {
239  EGL_WIDTH, buf_data->meta.width,
240  EGL_HEIGHT, buf_data->meta.height,
241  EGL_LINUX_DRM_FOURCC_EXT, buf_data->meta.fourcc,
242  EGL_DMA_BUF_PLANE0_FD_EXT, buf_data->fd,
243  EGL_DMA_BUF_PLANE0_OFFSET_EXT, buf_data->meta.offset,
244  EGL_DMA_BUF_PLANE0_PITCH_EXT, buf_data->meta.stride,
245  EGL_NONE
246  };
247 
248  buf_fd = buf_data->fd;
249  egl_image = _eglCreateImageKHR(egl_display, EGL_NO_CONTEXT,
250  EGL_LINUX_DMA_BUF_EXT, NULL, image_attrs);
251  if (egl_image == EGL_NO_IMAGE_KHR) {
252  MH_ERROR("eglCreateImageKHR error 0x%X", eglGetError());
253  return false;
254  }
255 
256  // TODO Do this when swapping if we end up importing more than one buffer
257  glBindTexture(GL_TEXTURE_2D, gl_texture);
258  _glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, egl_image);
259 
260  while((err = glGetError()) != GL_NO_ERROR)
261  MH_WARNING("OpenGL error 0x%X", err);
262 
263  MH_DEBUG("Image successfully imported");
264 
265  return true;
266  }
267 
268  uint32_t gl_texture;
269  promise<BufferData> prom_buff;
270  future<BufferData> fut_buff;
271  int sock_fd;
272  thread sock_thread;
273  EGLImageKHR egl_image;
274  int buf_fd;
275  PFNEGLCREATEIMAGEKHRPROC _eglCreateImageKHR;
276  PFNEGLDESTROYIMAGEKHRPROC _eglDestroyImageKHR;
277  PFNGLEGLIMAGETARGETTEXTURE2DOESPROC _glEGLImageTargetTexture2DOES;
278 };
279 
280 } // namespace MediaHub
281 } // namespace lomiri
282 
283 EglVideoSink::EglVideoSink(uint32_t gl_texture,
284  PlayerKey key,
285  QObject *parent):
286  VideoSink(new EglVideoSinkPrivate(gl_texture, key, this), parent)
287 {
288 }
289 
291 {
292 }
293 
295 {
296  return [playerKey](uint32_t textureId, QObject *parent) {
297  return new EglVideoSink(textureId, playerKey, parent);
298  };
299 }
300 
302 {
303  Q_D(EglVideoSink);
304 
305  // First time called, import buffers
306  if (d->egl_image == EGL_NO_IMAGE_KHR) {
307  BufferData buf_data = d->fut_buff.get();
308  if (!d->import_buffer(&buf_data))
309  return false;
310  }
311 
312  // We need to do nothing here, as the only buffer has already been mapped.
313  // TODO Change when we implement a buffer queue.
314 
315  return true;
316 }
lomiri::MediaHub
Definition: dbus_utils.h:25
lomiri::MediaHub::EglVideoSinkPrivate::buf_fd
int buf_fd
Definition: egl_video_sink.cpp:274
QObject
socket_types.h
lomiri::MediaHub::BufferData::fd
int fd
Definition: socket_types.h:36
lomiri::MediaHub::VideoSink::frameAvailable
void frameAvailable()
The signal is emitted whenever a new frame is available and a subsequent call to swapBuffers() will n...
lomiri::MediaHub::EglVideoSinkPrivate::_glEGLImageTargetTexture2DOES
PFNGLEGLIMAGETARGETTEXTURE2DOESPROC _glEGLImageTargetTexture2DOES
Definition: egl_video_sink.cpp:277
lomiri::MediaHub::EglVideoSinkPrivate::_eglDestroyImageKHR
PFNEGLDESTROYIMAGEKHRPROC _eglDestroyImageKHR
Definition: egl_video_sink.cpp:276
lomiri::MediaHub::EglVideoSinkPrivate::egl_image
EGLImageKHR egl_image
Definition: egl_video_sink.cpp:273
lomiri::MediaHub::EglVideoSinkPrivate
Definition: egl_video_sink.cpp:45
lomiri::MediaHub::BufferMetadata::width
int width
Definition: socket_types.h:27
lomiri::MediaHub::VideoSink
A video sink abstracts a queue of buffers, that receives a stream of decoded video buffers from an ar...
Definition: video_sink.h:34
lomiri::MediaHub::VideoSinkFactory
std::function< VideoSink *(uint32_t textureId, QObject *parent)> VideoSinkFactory
Definition: video_sink_p.h:43
lomiri::MediaHub::BufferMetadata::stride
int stride
Definition: socket_types.h:30
lomiri::MediaHub::BufferData
Definition: socket_types.h:34
lomiri::MediaHub::BufferMetadata::fourcc
int fourcc
Definition: socket_types.h:29
lomiri::MediaHub::EglVideoSinkPrivate::gl_texture
uint32_t gl_texture
Definition: egl_video_sink.cpp:268
lomiri::MediaHub::BufferData::meta
BufferMetadata meta
Definition: socket_types.h:37
MH_ERROR
#define MH_ERROR(...)
Definition: logging.h:41
lomiri::MediaHub::EglVideoSinkPrivate::prom_buff
promise< BufferData > prom_buff
Definition: egl_video_sink.cpp:269
lomiri::MediaHub::EglVideoSinkPrivate::sock_fd
int sock_fd
Definition: egl_video_sink.cpp:271
lomiri::MediaHub::EglVideoSinkPrivate::fut_buff
future< BufferData > fut_buff
Definition: egl_video_sink.cpp:270
lomiri::MediaHub::EglVideoSinkPrivate::import_buffer
bool import_buffer(const BufferData *buf_data)
Definition: egl_video_sink.cpp:234
egl_video_sink.h
lomiri::MediaHub::EglVideoSink::swapBuffers
bool swapBuffers() override
Releases the current buffer, and consumes the next buffer in the queue, making it available for consu...
Definition: egl_video_sink.cpp:301
lomiri::MediaHub::EglVideoSinkPrivate::_eglCreateImageKHR
PFNEGLCREATEIMAGEKHRPROC _eglCreateImageKHR
Definition: egl_video_sink.cpp:275
lomiri::MediaHub::EglVideoSink
Definition: egl_video_sink.h:27
lomiri::MediaHub::EglVideoSinkPrivate::~EglVideoSinkPrivate
~EglVideoSinkPrivate()
Definition: egl_video_sink.cpp:215
lomiri::MediaHub::PlayerKey
uint32_t PlayerKey
Definition: video_sink_p.h:44
MH_DEBUG
#define MH_DEBUG(...)
Definition: logging.h:38
lomiri::MediaHub::BufferMetadata::height
int height
Definition: socket_types.h:28
lomiri
Definition: dbus_utils.h:24
lomiri::MediaHub::VideoSinkPrivate
Definition: video_sink_p.h:49
lomiri::MediaHub::EglVideoSinkPrivate::EglVideoSinkPrivate
EglVideoSinkPrivate(uint32_t gl_texture, PlayerKey key, EglVideoSink *q)
Definition: egl_video_sink.cpp:156
lomiri::MediaHub::EglVideoSink::~EglVideoSink
virtual ~EglVideoSink()
Definition: egl_video_sink.cpp:290
logging.h
lomiri::MediaHub::EglVideoSink::createFactory
static VideoSinkFactory createFactory(PlayerKey playerKey)
Definition: egl_video_sink.cpp:294
lomiri::MediaHub::BufferMetadata::offset
int offset
Definition: socket_types.h:31
MH_WARNING
#define MH_WARNING(...)
Definition: logging.h:40
lomiri::MediaHub::EglVideoSinkPrivate::sock_thread
thread sock_thread
Definition: egl_video_sink.cpp:272