/* * Android MediaCodec implementation for Android Translation Layer * * Large part of the video decoding code was taken from https://git.sr.ht/~emersion/vaapi-decoder/tree/wayland/ */ #include #include #include #include #include #include #include #include #include #include #include #include #if !GTK_CHECK_VERSION(4, 14, 0) #include #include "linux-dmabuf-unstable-v1-client-protocol.h" #include "viewporter-client-protocol.h" #endif #include "jni.h" #include "../generated_headers/android_media_MediaCodec.h" #include "../util.h" #include "src/api-impl-jni/defines.h" #include "../../libandroid/native_window.h" struct ATL_codec_context { AVCodecContext *codec; union { struct { SwrContext *swr; } audio; struct { #if GTK_CHECK_VERSION(4, 14, 0) GtkPicture *gtk_picture; #else struct zwp_linux_dmabuf_v1 *zwp_linux_dmabuf_v1; struct wp_viewporter *wp_viewporter; struct wp_viewport *viewport; struct ANativeWindow *native_window; int surface_width; int surface_height; #endif } video; }; }; static enum AVPixelFormat hw_pix_fmt = AV_PIX_FMT_NONE; JNIEXPORT jlong JNICALL Java_android_media_MediaCodec_native_1constructor(JNIEnv *env, jobject this, jstring codec_name) { const char *name = (*env)->GetStringUTFChars(env, codec_name, NULL); const AVCodec *codec = avcodec_find_decoder_by_name(name); (*env)->ReleaseStringUTFChars(env, codec_name, name); AVCodecContext *codec_ctx = avcodec_alloc_context3(codec); struct ATL_codec_context *ctx = calloc(1, sizeof(struct ATL_codec_context)); ctx->codec = codec_ctx; return _INTPTR(ctx); } JNIEXPORT void JNICALL Java_android_media_MediaCodec_native_1configure_1audio(JNIEnv *env, jobject this, jlong codec, jobject extradata, jint sample_rate, jint nb_channels) { struct ATL_codec_context *ctx = _PTR(codec); AVCodecContext *codec_ctx = ctx->codec; jarray array_ref; jbyte *array; void *data; printf("Java_android_media_MediaCodec_native_1configure_1audio(%s, %d, %d)\n", codec_ctx->codec->name, sample_rate, nb_channels); codec_ctx->sample_rate = sample_rate; codec_ctx->ch_layout = (AVChannelLayout) AV_CHANNEL_LAYOUT_STEREO; codec_ctx->ch_layout.nb_channels = nb_channels; codec_ctx->extradata_size = get_nio_buffer_size(env, extradata); data = get_nio_buffer(env, extradata, &array_ref, &array); codec_ctx->extradata = av_mallocz(codec_ctx->extradata_size + AV_INPUT_BUFFER_PADDING_SIZE); memcpy(codec_ctx->extradata, data, codec_ctx->extradata_size); release_nio_buffer(env, array_ref, array); for (int i = 0; i < codec_ctx->extradata_size; i++) { printf("params->extradata[%d] = %x\n", i, codec_ctx->extradata[i]); } } /* * Helper functions for hardware accelerated video decoding using Wayland DMA-Buf protocol */ static const struct { uint32_t format; int nb_layers; uint32_t layers[AV_DRM_MAX_PLANES]; } drm_format_map[] = { { DRM_FORMAT_NV12, 2, { DRM_FORMAT_R8, DRM_FORMAT_GR88 } }, }; static uint32_t get_drm_frame_format(const AVDRMFrameDescriptor *drm_frame_desc) { if (drm_frame_desc->nb_layers == 1) { return drm_frame_desc->layers[0].format; } for (size_t i = 0; i < sizeof(drm_format_map) / sizeof(drm_format_map[0]); i++) { if (drm_format_map[i].nb_layers != drm_frame_desc->nb_layers) { continue; } int match = 1; for (int j = 0; j < drm_frame_desc->nb_layers; j++) { match &= drm_frame_desc->layers[j].format == drm_format_map[i].layers[j]; } if (match) { return drm_format_map[i].format; } } return DRM_FORMAT_INVALID; } static int check_hw_device_type(enum AVHWDeviceType type) { enum AVHWDeviceType t = AV_HWDEVICE_TYPE_NONE; while (1) { t = av_hwdevice_iterate_types(t); if (t == AV_HWDEVICE_TYPE_NONE) { break; } if (t == type) { return 0; } } return -1; } static enum AVPixelFormat get_hw_format(AVCodecContext *ctx, const enum AVPixelFormat *pix_fmts) { for (size_t i = 0; pix_fmts[i] != AV_PIX_FMT_NONE; i++) { if (pix_fmts[i] == hw_pix_fmt) { return hw_pix_fmt; } } fprintf(stderr, "Failed to find HW pixel format\n"); return AV_PIX_FMT_NONE; } struct render_frame_data { AVFrame *frame; AVFrame *drm_frame; #if GTK_CHECK_VERSION(4, 14, 0) GtkPicture *gtk_picture; #else struct zwp_linux_dmabuf_v1 *zwp_linux_dmabuf_v1; struct ANativeWindow *native_window; #endif }; #if GTK_CHECK_VERSION(4, 14, 0) static void handle_dmabuftexture_destroy(void *data) { struct render_frame_data *d = data; AVFrame *frame = d->frame; av_frame_free(&frame); AVFrame *drm_frame = d->drm_frame; av_frame_free(&drm_frame); free(d); } static GdkTexture *import_drm_frame_desc_as_texture(const AVDRMFrameDescriptor *drm_frame_desc, int width, int height, struct render_frame_data *d) { // VA-API drivers may use separate layers with one plane each, or a single // layer with multiple planes. We need to handle both. uint32_t drm_format = get_drm_frame_format(drm_frame_desc); if (drm_format == DRM_FORMAT_INVALID) { fprintf(stderr, "Failed to get DRM frame format\n"); return NULL; } GdkDmabufTextureBuilder* builder = gdk_dmabuf_texture_builder_new(); gdk_dmabuf_texture_builder_set_display(builder, gdk_display_get_default()); int k = 0; for (int i = 0; i < drm_frame_desc->nb_layers; i++) { const AVDRMLayerDescriptor *drm_layer = &drm_frame_desc->layers[i]; for (int j = 0; j < drm_layer->nb_planes; j++) { const AVDRMPlaneDescriptor *drm_plane = &drm_layer->planes[j]; const AVDRMObjectDescriptor *drm_object = &drm_frame_desc->objects[drm_plane->object_index]; gdk_dmabuf_texture_builder_set_modifier(builder, drm_object->format_modifier); gdk_dmabuf_texture_builder_set_offset(builder, k, drm_plane->offset); gdk_dmabuf_texture_builder_set_stride(builder, k, drm_plane->pitch); gdk_dmabuf_texture_builder_set_fd(builder, k, drm_object->fd); k++; } } gdk_dmabuf_texture_builder_set_n_planes(builder, k); gdk_dmabuf_texture_builder_set_width(builder, width); gdk_dmabuf_texture_builder_set_height(builder, height); gdk_dmabuf_texture_builder_set_fourcc(builder, drm_format); GError *error = NULL; GdkTexture *texture = gdk_dmabuf_texture_builder_build(builder, handle_dmabuftexture_destroy, d, &error); if (error) { fprintf(stderr, "Failed to build texture: %s\n", error->message); exit(1); } g_object_unref(builder); return texture; } #else // GTK_CHECK_VERSION(4, 14, 0) static void handle_global(void *data, struct wl_registry *registry, uint32_t name, const char *interface, uint32_t version) { struct ATL_codec_context *ctx = data; if (strcmp(interface, zwp_linux_dmabuf_v1_interface.name) == 0) { ctx->video.zwp_linux_dmabuf_v1 = wl_registry_bind(registry, name, &zwp_linux_dmabuf_v1_interface, 2); } else if (strcmp(interface, wp_viewporter_interface.name) == 0) { ctx->video.wp_viewporter = wl_registry_bind(registry, name, &wp_viewporter_interface, 1); } } static void handle_global_remove(void *data, struct wl_registry *registry, uint32_t name) { // This space is intentionally left blank } static const struct wl_registry_listener registry_listener = { .global = handle_global, .global_remove = handle_global_remove, }; static struct wl_buffer *import_drm_frame_desc(struct zwp_linux_dmabuf_v1 *zwp_linux_dmabuf_v1, const AVDRMFrameDescriptor *drm_frame_desc, int width, int height) { // VA-API drivers may use separate layers with one plane each, or a single // layer with multiple planes. We need to handle both. uint32_t drm_format = get_drm_frame_format(drm_frame_desc); if (drm_format == DRM_FORMAT_INVALID) { fprintf(stderr, "Failed to get DRM frame format\n"); return NULL; } // fprintf(stderr, "DRM format: 0x%X\n", drm_format); struct zwp_linux_buffer_params_v1 *dmabuf_params = zwp_linux_dmabuf_v1_create_params(zwp_linux_dmabuf_v1); int k = 0; for (int i = 0; i < drm_frame_desc->nb_layers; i++) { const AVDRMLayerDescriptor *drm_layer = &drm_frame_desc->layers[i]; for (int j = 0; j < drm_layer->nb_planes; j++) { const AVDRMPlaneDescriptor *drm_plane = &drm_layer->planes[j]; const AVDRMObjectDescriptor *drm_object = &drm_frame_desc->objects[drm_plane->object_index]; uint32_t modifier_hi = drm_object->format_modifier >> 32; uint32_t modifier_lo = drm_object->format_modifier & 0xFFFFFFFF; zwp_linux_buffer_params_v1_add(dmabuf_params, drm_object->fd, k, drm_plane->offset, drm_plane->pitch, modifier_hi, modifier_lo); k++; } } return zwp_linux_buffer_params_v1_create_immed(dmabuf_params, width, height, drm_format, 0); } static void handle_buffer_release(void *data, struct wl_buffer *buffer) { AVFrame *frame = data; av_frame_free(&frame); wl_buffer_destroy(buffer); } static const struct wl_buffer_listener buffer_listener = { .release = handle_buffer_release, }; static void on_resize(GtkWidget* widget, gint width, gint height, struct ATL_codec_context *ctx) { ctx->video.surface_width = gtk_widget_get_width(widget); ctx->video.surface_height = gtk_widget_get_height(widget); wp_viewport_set_destination(ctx->video.viewport, ctx->video.surface_width, ctx->video.surface_height); } #endif // GTK_CHECK_VERSION(4, 14, 0) JNIEXPORT void JNICALL Java_android_media_MediaCodec_native_1configure_1video(JNIEnv *env, jobject this, jlong codec, jobject csd0, jobject csd1, jobject surface_obj) { struct ATL_codec_context *ctx = _PTR(codec); AVCodecContext *codec_ctx = ctx->codec; jarray array_ref; jbyte *array; void *data; int sps_size; int pps_size; printf("Java_android_media_MediaCodec_native_1configure_video(%s)\n", codec_ctx->codec->name); sps_size = get_nio_buffer_size(env, csd0); pps_size = get_nio_buffer_size(env, csd1); // libavcodec wants the complete AVC decoder configuration record, but android APIs pass only SPS and PPS records // see https://stackoverflow.com/questions/29790334/how-to-fill-extradata-field-of-avcodeccontext-with-sps-and-pps-data codec_ctx->extradata_size = 11 + sps_size + pps_size; codec_ctx->extradata = av_mallocz(codec_ctx->extradata_size + AV_INPUT_BUFFER_PADDING_SIZE); codec_ctx->extradata[0] = 0x01; codec_ctx->extradata[1] = 0x4d; codec_ctx->extradata[2] = 0x40; codec_ctx->extradata[3] = 0x1f; codec_ctx->extradata[4] = 0xff; codec_ctx->extradata[5] = 0xe0 | 1; codec_ctx->extradata[6] = (sps_size>>8) & 0xff; codec_ctx->extradata[7] = (sps_size) & 0xff; data = get_nio_buffer(env, csd0, &array_ref, &array); memcpy(codec_ctx->extradata + 8, data, sps_size); release_nio_buffer(env, array_ref, array); codec_ctx->extradata[8 + sps_size] = 1; codec_ctx->extradata[9 + sps_size] = (pps_size>>8) & 0xff; codec_ctx->extradata[10 + sps_size] = (pps_size) & 0xff; data = get_nio_buffer(env, csd1, &array_ref, &array); memcpy(codec_ctx->extradata + 11 + sps_size, data, pps_size); release_nio_buffer(env, array_ref, array); for (int i = 0; i < codec_ctx->extradata_size; i++) { printf("params->extradata[%d] = %x\n", i, codec_ctx->extradata[i]); } enum AVHWDeviceType type = AV_HWDEVICE_TYPE_VAAPI; int ret = check_hw_device_type(type); if (ret != 0) { fprintf(stderr, "VA-API not supported\n"); exit(1); } int i = 0; while (1) { const AVCodecHWConfig *config = avcodec_get_hw_config(codec_ctx->codec, i); if (!config) { fprintf(stderr, "Decoder %s doesn't support device type %s\n", codec_ctx->codec->name, av_hwdevice_get_type_name(type)); exit(1); } if ((config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) && config->device_type == type) { hw_pix_fmt = config->pix_fmt; break; } i++; } fprintf(stderr, "Selected pixel format %s\n", av_get_pix_fmt_name(hw_pix_fmt)); #if GTK_CHECK_VERSION(4, 14, 0) GtkWidget *surface_view_widget = _PTR(_GET_LONG_FIELD(surface_obj, "widget")); GtkWidget *graphics_offload = gtk_widget_get_first_child(surface_view_widget); if (!GTK_IS_GRAPHICS_OFFLOAD(graphics_offload)) { graphics_offload = gtk_graphics_offload_new(gtk_picture_new()); gtk_widget_insert_after(graphics_offload, surface_view_widget, NULL); } ctx->video.gtk_picture = GTK_PICTURE(gtk_graphics_offload_get_child(GTK_GRAPHICS_OFFLOAD(graphics_offload))); #else struct ANativeWindow *native_window = ANativeWindow_fromSurface(env, surface_obj); ctx->video.native_window = native_window; ctx->video.surface_width = gtk_widget_get_width(native_window->surface_view_widget); ctx->video.surface_height = gtk_widget_get_height(native_window->surface_view_widget); #endif } JNIEXPORT void JNICALL Java_android_media_MediaCodec_native_1start(JNIEnv *env, jobject this, jlong codec) { struct ATL_codec_context *ctx = _PTR(codec); AVCodecContext *codec_ctx = ctx->codec; if(avcodec_open2(codec_ctx, codec_ctx->codec, NULL)<0){ printf("Codec cannot be found"); } if (codec_ctx->codec_type == AVMEDIA_TYPE_AUDIO) { printf("ctx->sample_rate = %d\n", codec_ctx->sample_rate); printf("ctx->ch_layout.nb_channels = %d\n", codec_ctx->ch_layout.nb_channels); printf("ctx->sample_fmt = %d\n", codec_ctx->sample_fmt); int ret = swr_alloc_set_opts2(&ctx->audio.swr, &codec_ctx->ch_layout, AV_SAMPLE_FMT_S16, codec_ctx->sample_rate, &codec_ctx->ch_layout, codec_ctx->sample_fmt, codec_ctx->sample_rate, 0, NULL); if (ret != 0) { fprintf(stderr, "FFmpegDecoder error: Swresampler alloc fail\n"); } swr_init(ctx->audio.swr); } else if (codec_ctx->codec_type == AVMEDIA_TYPE_VIDEO) { enum AVHWDeviceType type = AV_HWDEVICE_TYPE_VAAPI; codec_ctx->get_format = get_hw_format; AVBufferRef *hw_device_ctx = NULL; int ret = av_hwdevice_ctx_create(&hw_device_ctx, type, NULL, NULL, 0); if (ret < 0) { fprintf(stderr, "Failed to create HW device context\n"); exit(1); } codec_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx); #if !GTK_CHECK_VERSION(4, 14, 0) struct ANativeWindow *native_window = ctx->video.native_window; struct wl_registry *registry = wl_display_get_registry(native_window->wayland_display); wl_registry_add_listener(registry, ®istry_listener, ctx); wl_display_roundtrip(native_window->wayland_display); wl_registry_destroy(registry); if (ctx->video.zwp_linux_dmabuf_v1 == NULL || ctx->video.wp_viewporter == NULL) { fprintf(stderr, "Missing zwp_linux_dmabuf_v1 or wp_viewporter support\n"); exit(1); } ctx->video.viewport = wp_viewporter_get_viewport(ctx->video.wp_viewporter, native_window->wayland_surface); wp_viewport_set_destination(ctx->video.viewport, ctx->video.surface_width, ctx->video.surface_height); g_signal_connect(native_window->surface_view_widget, "resize", G_CALLBACK(on_resize), ctx); #endif } } #define INFO_TRY_AGAIN_LATER -1 JNIEXPORT jint JNICALL Java_android_media_MediaCodec_native_1queueInputBuffer(JNIEnv *env, jobject this, jlong codec, jobject buffer, jlong presentationTimeUs) { jarray array_ref; jbyte *array; int ret; struct ATL_codec_context *ctx = _PTR(codec); AVCodecContext *codec_ctx = ctx->codec; AVPacket *pkt = av_packet_alloc(); pkt->size = get_nio_buffer_size(env, buffer); pkt->data = get_nio_buffer(env, buffer, &array_ref, &array); pkt->pts = presentationTimeUs; ret = avcodec_send_packet(codec_ctx, pkt); if (ret < 0 && ret != AVERROR(EAGAIN)) { fprintf(stderr, "Error while sending packet: %d = %s\n", ret, av_err2str(ret)); } release_nio_buffer(env, array_ref, array); av_packet_free(&pkt); return ret; } JNIEXPORT jint JNICALL Java_android_media_MediaCodec_native_1dequeueOutputBuffer(JNIEnv *env, jobject this, jlong codec, jobject buffer, jobject buffer_info) { struct ATL_codec_context *ctx = _PTR(codec); AVCodecContext *codec_ctx = ctx->codec; AVFrame *frame = av_frame_alloc(); int ret; jarray array_ref; jbyte *array; ret = avcodec_receive_frame(codec_ctx, frame); if (ret < 0) { if (ret != AVERROR(EAGAIN)) { printf("avcodec_receive_frame returned %d\n", ret); printf("frame->data = %p frame->nb_samples = %d\n", frame->data[0], frame->nb_samples); } av_frame_free(&frame); return INFO_TRY_AGAIN_LATER; } _SET_LONG_FIELD(buffer_info, "presentationTimeUs", frame->pts); if (codec_ctx->codec_type == AVMEDIA_TYPE_AUDIO) { uint8_t *raw_buffer = get_nio_buffer(env, buffer, &array_ref, &array); int outSamples = swr_convert(ctx->audio.swr, &raw_buffer, frame->nb_samples, (uint8_t const **) (frame->data), frame->nb_samples); release_nio_buffer(env, array_ref, array); _SET_INT_FIELD(buffer_info, "offset", 0); _SET_INT_FIELD(buffer_info, "size", outSamples * 2 * 2); av_frame_free(&frame); } else if (codec_ctx->codec_type == AVMEDIA_TYPE_VIDEO) { // copy frame pointer into data buffer to be read by releaseOutputBuffer function uint8_t *raw_buffer = get_nio_buffer(env, buffer, &array_ref, &array); *((AVFrame **)raw_buffer) = frame; release_nio_buffer(env, array_ref, array); _SET_INT_FIELD(buffer_info, "offset", 0); _SET_INT_FIELD(buffer_info, "size", sizeof(AVFrame *)); } return 0; } // callback to perform wayland stuff on main thread static gboolean render_frame(void *data) { struct render_frame_data *d = (struct render_frame_data *)data; AVFrame *frame = d->frame; int ret; AVFrame *drm_frame = d->drm_frame = av_frame_alloc(); drm_frame->format = AV_PIX_FMT_DRM_PRIME; drm_frame->hw_frames_ctx = av_buffer_ref(frame->hw_frames_ctx); // Convert the VA-API frame into a DMA-BUF frame ret = av_hwframe_map(drm_frame, frame, 0); if (ret < 0) { fprintf(stderr, "Failed to map frame: %s\n", av_err2str(ret)); exit(1); } AVDRMFrameDescriptor *drm_frame_desc = (void *)drm_frame->data[0]; #if GTK_CHECK_VERSION(4, 14, 0) GdkTexture *texture = import_drm_frame_desc_as_texture(drm_frame_desc, drm_frame->width, drm_frame->height, d); gtk_picture_set_paintable(d->gtk_picture, GDK_PAINTABLE(texture)); g_object_unref(texture); #else struct wl_buffer *wl_buffer = import_drm_frame_desc(d->zwp_linux_dmabuf_v1, drm_frame_desc, drm_frame->width, drm_frame->height); if (!wl_buffer) { exit(1); } wl_buffer_add_listener(wl_buffer, &buffer_listener, frame); struct ANativeWindow *native_window = d->native_window; wl_surface_damage(native_window->wayland_surface, 0, 0, INT32_MAX, INT32_MAX); wl_surface_attach(native_window->wayland_surface, wl_buffer, 0, 0); wl_surface_commit(native_window->wayland_surface); // actual frame will be freed in handle_buffer_release callback av_frame_free(&drm_frame); free(d); #endif return G_SOURCE_REMOVE; } JNIEXPORT void JNICALL Java_android_media_MediaCodec_native_1releaseOutputBuffer(JNIEnv *env, jobject this, jlong codec, jobject buffer, jboolean render) { struct ATL_codec_context *ctx = _PTR(codec); jarray array_ref; jbyte *array; AVFrame *frame; if (ctx->codec->codec_type == AVMEDIA_TYPE_VIDEO) { AVFrame **raw_buffer = get_nio_buffer(env, buffer, &array_ref, &array); frame = *raw_buffer; *raw_buffer = NULL; release_nio_buffer(env, array_ref, array); if (!render) { fprintf(stderr, "skipping %dx%d frame!\n", frame->width, frame->height); av_frame_free(&frame); return; } struct render_frame_data *data = malloc(sizeof(struct render_frame_data)); data->frame = frame; #if GTK_CHECK_VERSION(4, 14, 0) data->gtk_picture = ctx->video.gtk_picture; #else data->native_window = ctx->video.native_window; data->zwp_linux_dmabuf_v1 = ctx->video.zwp_linux_dmabuf_v1; #endif g_idle_add(render_frame, data); } } JNIEXPORT void JNICALL Java_android_media_MediaCodec_native_1release(JNIEnv *env, jobject this, jlong codec) { struct ATL_codec_context *ctx = _PTR(codec); avcodec_free_context(&ctx->codec); free(ctx); }