Decode animation frames and pass FrameInfos to dart (#4324)

https://github.com/flutter/flutter/issues/204

lib/ui/BUILD.gn

@@ -18,6 +18,8 @@ source_set("ui") {
     "painting/canvas.h",
     "painting/codec.cc",
     "painting/codec.h",
+    "painting/frame_info.cc",
+    "painting/frame_info.h",
     "painting/gradient.cc",
     "painting/gradient.h",
     "painting/image.cc",

lib/ui/dart_ui.cc

@@ -9,6 +9,7 @@
 #include "flutter/lib/ui/dart_runtime_hooks.h"
 #include "flutter/lib/ui/painting/canvas.h"
 #include "flutter/lib/ui/painting/codec.h"
+#include "flutter/lib/ui/painting/frame_info.h"
 #include "flutter/lib/ui/painting/gradient.h"
 #include "flutter/lib/ui/painting/image.h"
 #include "flutter/lib/ui/painting/image_decoding.h"
@@ -56,6 +57,7 @@ void DartUI::InitForGlobal() {
     CanvasPath::RegisterNatives(g_natives);
     Codec::RegisterNatives(g_natives);
     DartRuntimeHooks::RegisterNatives(g_natives);
+    FrameInfo::RegisterNatives(g_natives);
     ImageDecoding::RegisterNatives(g_natives);
     ImageFilter::RegisterNatives(g_natives);
     ImageShader::RegisterNatives(g_natives);

lib/ui/painting.dart

@@ -879,6 +879,9 @@ abstract class FrameInfo extends NativeFieldWrapperClass2 {
   Image get image native "FrameInfo_image";
 }
 
+/// Callback signature for [Codec.getNextFrame].
+typedef void NextFrameInfoCallback(FrameInfo frameInfo);
+
 /// A handle to an image codec.
 abstract class Codec extends NativeFieldWrapperClass2 {
   /// Number of frames in this image.
@@ -890,10 +893,12 @@ abstract class Codec extends NativeFieldWrapperClass2 {
   /// * -1 for infinity repetitions.
   int get repetitionCount native "Codec_repetitionCount";
 
-  /// Returns the next animation frame.
+  /// Fetches the next animation frame.
   ///
   /// Wraps back to the first frame after returning the last frame.
-  FrameInfo getNextFrame() native "Codec_getNextFrame";
+  ///
+  /// Returns an error message on failure, null on success.
+  String getNextFrame(NextFrameInfoCallback callback) native "Codec_getNextFrame";
 
   /// Release the resources used by this object. The object is no longer usable
   /// after this method is called.

lib/ui/painting/codec.cc

@@ -6,6 +6,7 @@
 
 #include "flutter/common/threads.h"
 #include "flutter/glue/trace_event.h"
+#include "flutter/lib/ui/painting/frame_info.h"
 #include "lib/fxl/functional/make_copyable.h"
 #include "lib/tonic/dart_binding_macros.h"
 #include "lib/tonic/dart_library_natives.h"
@@ -20,28 +21,10 @@ using tonic::ToDart;
 
 namespace blink {
 
-IMPLEMENT_WRAPPERTYPEINFO(ui, Codec);
-
-#define FOR_EACH_BINDING(V) \
-  V(Codec, frameCount)      \
-  V(Codec, repetitionCount) \
-  V(Codec, dispose)
-
-FOR_EACH_BINDING(DART_NATIVE_CALLBACK)
-
-void Codec::dispose() {
-  ClearDartWrapper();
-}
-
-MultiFrameCodec::MultiFrameCodec(std::unique_ptr<SkCodec> codec)
-    : codec_(std::move(codec)) {
-  frameCount_ = codec_->getFrameCount();
-  repetitionCount_ = codec_->getRepetitionCount();
-}
-
 namespace {
 
 static constexpr const char* kInitCodecTraceTag = "InitCodec";
+static constexpr const char* kCodecNextFrameTraceTag = "CodecNextFrame";
 
 std::unique_ptr<SkCodec> InitCodec(sk_sp<SkData> buffer, size_t trace_id) {
   TRACE_FLOW_STEP("flutter", kInitCodecTraceTag, trace_id);
@@ -119,8 +102,153 @@ void InstantiateImageCodec(Dart_NativeArguments args) {
   }));
 }
 
+bool copy_to(SkBitmap* dst, SkColorType dstColorType, const SkBitmap& src) {
+  SkPixmap srcPM;
+  if (!src.peekPixels(&srcPM)) {
+    return false;
+  }
+
+  SkBitmap tmpDst;
+  SkImageInfo dstInfo = srcPM.info().makeColorType(dstColorType);
+  if (!tmpDst.setInfo(dstInfo)) {
+    return false;
+  }
+
+  if (!tmpDst.tryAllocPixels()) {
+    return false;
+  }
+
+  SkPixmap dstPM;
+  if (!tmpDst.peekPixels(&dstPM)) {
+    return false;
+  }
+
+  if (!srcPM.readPixels(dstPM)) {
+    return false;
+  }
+
+  dst->swap(tmpDst);
+  return true;
+}
+
+void InvokeNextFrameCallback(fxl::RefPtr<FrameInfo> frameInfo,
+                             std::unique_ptr<DartPersistentValue> callback,
+                             size_t trace_id) {
+  tonic::DartState* dart_state = callback->dart_state().get();
+  if (!dart_state) {
+    TRACE_FLOW_END("flutter", kCodecNextFrameTraceTag, trace_id);
+    return;
+  }
+  tonic::DartState::Scope scope(dart_state);
+  if (!frameInfo) {
+    DartInvoke(callback->value(), {Dart_Null()});
+  } else {
+    DartInvoke(callback->value(), {ToDart(frameInfo)});
+  }
+  TRACE_FLOW_END("flutter", kCodecNextFrameTraceTag, trace_id);
+}
+
 }  // namespace
 
+IMPLEMENT_WRAPPERTYPEINFO(ui, Codec);
+
+#define FOR_EACH_BINDING(V) \
+  V(Codec, getNextFrame)    \
+  V(Codec, frameCount)      \
+  V(Codec, repetitionCount) \
+  V(Codec, dispose)
+
+FOR_EACH_BINDING(DART_NATIVE_CALLBACK)
+
+void Codec::dispose() {
+  ClearDartWrapper();
+}
+
+MultiFrameCodec::MultiFrameCodec(std::unique_ptr<SkCodec> codec)
+    : codec_(std::move(codec)) {
+  repetitionCount_ = codec_->getRepetitionCount();
+  frameInfos_ = codec_->getFrameInfo();
+  frameBitmaps_.resize(frameInfos_.size());
+  nextFrameIndex_ = 0;
+}
+
+sk_sp<SkImage> MultiFrameCodec::GetNextFrameImage() {
+  SkBitmap& bitmap = frameBitmaps_[nextFrameIndex_];
+  if (!bitmap.getPixels()) {  // We haven't decoded this frame yet
+    const SkImageInfo info = codec_->getInfo().makeColorType(kN32_SkColorType);
+    bitmap.allocPixels(info);
+
+    SkCodec::Options options;
+    options.fFrameIndex = nextFrameIndex_;
+    const int requiredFrame = frameInfos_[nextFrameIndex_].fRequiredFrame;
+    if (requiredFrame != SkCodec::kNone) {
+      if (requiredFrame < 0 ||
+          static_cast<size_t>(requiredFrame) >= frameBitmaps_.size()) {
+        FXL_LOG(ERROR) << "Frame " << nextFrameIndex_ << " depends on frame "
+                       << requiredFrame << " which out of range (0,"
+                       << frameBitmaps_.size() << ").";
+        return NULL;
+      }
+      SkBitmap& requiredBitmap = frameBitmaps_[requiredFrame];
+      // For simplicity, do not try to cache old frames
+      if (requiredBitmap.getPixels() &&
+          copy_to(&bitmap, requiredBitmap.colorType(), requiredBitmap)) {
+        options.fPriorFrame = requiredFrame;
+      }
+    }
+
+    if (SkCodec::kSuccess != codec_->getPixels(info, bitmap.getPixels(),
+                                               bitmap.rowBytes(), &options)) {
+      FXL_LOG(ERROR) << "Could not getPixels for frame " << nextFrameIndex_;
+      return NULL;
+    }
+  }
+
+  return SkImage::MakeFromBitmap(bitmap);
+}
+
+void MultiFrameCodec::GetNextFrameAndInvokeCallback(
+    std::unique_ptr<DartPersistentValue> callback,
+    size_t trace_id) {
+  fxl::RefPtr<FrameInfo> frameInfo = NULL;
+  sk_sp<SkImage> skImage = GetNextFrameImage();
+  if (skImage) {
+    fxl::RefPtr<CanvasImage> image = CanvasImage::Create();
+    image->set_image(skImage);
+    frameInfo = fxl::MakeRefCounted<FrameInfo>(
+        std::move(image), frameInfos_[nextFrameIndex_].fDuration);
+  }
+  nextFrameIndex_ = (nextFrameIndex_ + 1) % frameInfos_.size();
+
+  Threads::UI()->PostTask(fxl::MakeCopyable(
+      [ callback = std::move(callback), frameInfo, trace_id ]() mutable {
+        InvokeNextFrameCallback(frameInfo, std::move(callback), trace_id);
+      }));
+
+  TRACE_FLOW_END("flutter", kCodecNextFrameTraceTag, trace_id);
+}
+
+Dart_Handle MultiFrameCodec::getNextFrame(Dart_Handle callback_handle) {
+  static size_t trace_counter = 1;
+  const size_t trace_id = trace_counter++;
+  TRACE_FLOW_BEGIN("flutter", kCodecNextFrameTraceTag, trace_id);
+
+  if (!Dart_IsClosure(callback_handle)) {
+    TRACE_FLOW_END("flutter", kCodecNextFrameTraceTag, trace_id);
+    return ToDart("Callback must be a function");
+  }
+
+  Threads::IO()->PostTask(fxl::MakeCopyable([
+    callback = std::make_unique<DartPersistentValue>(
+        tonic::DartState::Current(), callback_handle),
+    this, trace_id
+  ]() mutable {
+    GetNextFrameAndInvokeCallback(std::move(callback), trace_id);
+  }));
+
+  return Dart_Null();
+}
+
 void Codec::RegisterNatives(tonic::DartLibraryNatives* natives) {
   natives->Register({
       {"instantiateImageCodec", InstantiateImageCodec, 2, true},

lib/ui/painting/codec.h

@@ -7,6 +7,10 @@
 
 #include "lib/tonic/dart_wrappable.h"
 #include "third_party/skia/include/codec/SkCodec.h"
+#include "third_party/skia/include/core/SkBitmap.h"
+#include "third_party/skia/include/core/SkImage.h"
+
+using tonic::DartPersistentValue;
 
 namespace tonic {
 class DartLibraryNatives;
@@ -24,6 +28,7 @@ class Codec : public fxl::RefCountedThreadSafe<Codec>,
  public:
   virtual int frameCount() = 0;
   virtual int repetitionCount() = 0;
+  virtual Dart_Handle getNextFrame(Dart_Handle callback_handle) = 0;
   void dispose();
 
   static void RegisterNatives(tonic::DartLibraryNatives* natives);
@@ -31,8 +36,9 @@ class Codec : public fxl::RefCountedThreadSafe<Codec>,
 
 class MultiFrameCodec : public Codec {
  public:
-  int frameCount() { return frameCount_; }
+  int frameCount() { return frameInfos_.size(); }
   int repetitionCount() { return repetitionCount_; }
+  Dart_Handle getNextFrame(Dart_Handle args);
 
   static void RegisterNatives(tonic::DartLibraryNatives* natives);
 
@@ -40,9 +46,17 @@ class MultiFrameCodec : public Codec {
   MultiFrameCodec(std::unique_ptr<SkCodec> codec);
   ~MultiFrameCodec() {}
 
+  sk_sp<SkImage> GetNextFrameImage();
+  void GetNextFrameAndInvokeCallback(
+      std::unique_ptr<DartPersistentValue> callback,
+      size_t trace_id);
+
   const std::unique_ptr<SkCodec> codec_;
-  int frameCount_;
   int repetitionCount_;
+  int nextFrameIndex_;
+
+  std::vector<SkCodec::FrameInfo> frameInfos_;
+  std::vector<SkBitmap> frameBitmaps_;
 
   FRIEND_MAKE_REF_COUNTED(MultiFrameCodec);
   FRIEND_REF_COUNTED_THREAD_SAFE(MultiFrameCodec);

lib/ui/painting/frame_info.cc

@@ -0,0 +1,25 @@
+
+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "flutter/lib/ui/painting/frame_info.h"
+
+#include "lib/tonic/dart_binding_macros.h"
+#include "lib/tonic/dart_library_natives.h"
+
+namespace blink {
+
+IMPLEMENT_WRAPPERTYPEINFO(ui, FrameInfo);
+
+#define FOR_EACH_BINDING(V)    \
+  V(FrameInfo, durationMillis) \
+  V(FrameInfo, image)
+
+FOR_EACH_BINDING(DART_NATIVE_CALLBACK)
+
+void FrameInfo::RegisterNatives(tonic::DartLibraryNatives* natives) {
+  natives->Register({FOR_EACH_BINDING(DART_REGISTER_NATIVE)});
+}
+
+}  // namespace blink

lib/ui/painting/frame_info.h

@@ -0,0 +1,42 @@
+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef FLUTTER_LIB_UI_PAINTING_FRAME_INFO_H_
+#define FLUTTER_LIB_UI_PAINTING_FRAME_INFO_H_
+
+#include "flutter/lib/ui/painting/image.h"
+#include "lib/tonic/dart_wrappable.h"
+
+namespace tonic {
+class DartLibraryNatives;
+}  // namespace tonic
+
+namespace blink {
+
+// A single animation frame.
+class FrameInfo final : public fxl::RefCountedThreadSafe<FrameInfo>,
+                        public tonic::DartWrappable {
+  DEFINE_WRAPPERTYPEINFO();
+
+ public:
+  int durationMillis() { return durationMillis_; }
+  fxl::RefPtr<CanvasImage> image() { return image_; }
+
+  static void RegisterNatives(tonic::DartLibraryNatives* natives);
+
+ private:
+  FrameInfo(fxl::RefPtr<CanvasImage> image, int durationMillis)
+      : image_(std::move(image)), durationMillis_(durationMillis) {}
+  ~FrameInfo(){};
+
+  const fxl::RefPtr<CanvasImage> image_;
+  const int durationMillis_;
+
+  FRIEND_MAKE_REF_COUNTED(FrameInfo);
+  FRIEND_REF_COUNTED_THREAD_SAFE(FrameInfo);
+};
+
+}  // namespace blink
+
+#endif  // FLUTTER_LIB_UI_PAINTING_FRAME_INFO_H_

testing/dart/codec_test.dart

@@ -41,6 +41,39 @@ void main() {
     ui.Codec codec = await completer.future;
     expect(codec, null);
   });
+
+  test('nextFrame fails when no callback provided', () async {
+    Uint8List data = await _getSkiaResource('alphabetAnim.gif').readAsBytes();
+    Completer<ui.Codec> completer = new Completer<ui.Codec>();
+    expect(ui.instantiateImageCodec(data, completer.complete), null);
+    ui.Codec codec = await completer.future;
+    expect(codec.getNextFrame(null), 'Callback must be a function');
+  });
+
+  test('nextFrame', () async {
+    Uint8List data = await _getSkiaResource('test640x479.gif').readAsBytes();
+    Completer<ui.Codec> completer = new Completer<ui.Codec>();
+    expect(ui.instantiateImageCodec(data, completer.complete), null);
+    ui.Codec codec = await completer.future;
+    List<List<int>> decodedFrameInfos = [];
+    for (int i = 0; i < 5; i++) {
+      Completer<ui.FrameInfo> frameCompleter = new Completer<ui.FrameInfo>();
+      codec.getNextFrame(frameCompleter.complete);
+      ui.FrameInfo frameInfo = await frameCompleter.future;
+      decodedFrameInfos.add([
+        frameInfo.durationMillis,
+        frameInfo.image.width,
+        frameInfo.image.height,
+      ]);
+    }
+    expect(decodedFrameInfos, equals([
+      [200, 640, 479],
+      [200, 640, 479],
+      [200, 640, 479],
+      [200, 640, 479],
+      [200, 640, 479],
+    ]));
+  });
 }
 
 /// Returns a File handle to a file in the skia/resources directory.

travis/licenses_golden/licenses_flutter

@@ -1139,6 +1139,8 @@ FILE: ../../../flutter/lib/ui/compositing/scene_host.cc
 FILE: ../../../flutter/lib/ui/compositing/scene_host.h
 FILE: ../../../flutter/lib/ui/painting/codec.cc
 FILE: ../../../flutter/lib/ui/painting/codec.h
+FILE: ../../../flutter/lib/ui/painting/frame_info.cc
+FILE: ../../../flutter/lib/ui/painting/frame_info.h
 FILE: ../../../flutter/lib/ui/painting/utils.cc
 FILE: ../../../flutter/lib/ui/painting/vertices.cc
 FILE: ../../../flutter/lib/ui/painting/vertices.h