Flutter实战图片组件演进之外接纹理解析
站长
· 阅读数 9
前言
其实Flutter本身已具备加载图片的能力，Image组件就满足网络图片、本地图片、文件图片的加载。那为什么我们还需要实现其他图片加载方案呢？其实是因为Flutter图片组件功能上存在一些缺陷：
- 图片缓存没有持久化能力，无网环境下不支持显示图片。
- 文件图片与原生环境不共用，导致图片资源文件重复。
因此为了满足日常开发需要和优化点，可以做点什么让图片组件功能达到满意的效果。接下来从Flutter原生组件再到外接纹理慢慢了解图片组件功能演进的过程。
Flutter原生图片组件
Flutter原生图片支持多种加载形式：
- Image.network（网络图片）
- Image.file （本地图片）
- Image.asset （文件图片）
- Image.memory （byte图片）
图片加载流程简要（网络图片为例）
- 第一步：网络图片加载形式以NetworkImage，内部由network_image.NetworkImage构成。
Image.network(
String src, {
......省略不必要代码
}) : image = ResizeImage.resizeIfNeeded(cacheWidth, cacheHeight, NetworkImage(src, scale: scale, headers: headers)),
...... 省略不必要代码
super(key: key);
......
const factory NetworkImage(String url, { double scale, Map<String, String> headers }) = network_image.NetworkImage;
- 第二步：NetWorkImage实质上继承于ImageProvider，其他加载形式也是如此。ImageProvider是处理图片基本抽象类，继承它的加载类主要实现load方法执行不同形式加载过程。
abstract class ImageProvider<T> {
const ImageProvider();
.......
@protected
ImageStreamCompleter load(T key, DecoderCallback decode);
.......
}
- 第三步：例如网络形式获取图片数据过程通过网络，通过Dart层网络请求HttpClient请求图片获取最终数据Uint8List。
class NetworkImage extends image_provider.ImageProvider<image_provider.NetworkImage> implements image_provider.NetworkImage {
.......
@override
ImageStreamCompleter load(image_provider.NetworkImage key, image_provider.DecoderCallback decode) {
final StreamController<ImageChunkEvent> chunkEvents = StreamController<ImageChunkEvent>();
return MultiFrameImageStreamCompleter(
codec: _loadAsync(key as NetworkImage, chunkEvents, decode),
chunkEvents: chunkEvents.stream,
scale: key.scale,
informationCollector: () {
return <DiagnosticsNode>[
DiagnosticsProperty<image_provider.ImageProvider>('Image provider', this),
DiagnosticsProperty<image_provider.NetworkImage>('Image key', key),
];
},
);
}
Future<ui.Codec> _loadAsync(
NetworkImage key,
StreamController<ImageChunkEvent> chunkEvents,
image_provider.DecoderCallback decode,
) async {
try {
final Uri resolved = Uri.base.resolve(key.url);
final HttpClientRequest request = await _httpClient.getUrl(resolved);
headers?.forEach((String name, String value) {
request.headers.add(name, value);
});
final HttpClientResponse response = await request.close();
if (response.statusCode != HttpStatus.ok) {
PaintingBinding.instance.imageCache.evict(key);
throw image_provider.NetworkImageLoadException(statusCode: response.statusCode, uri: resolved);
}
final Uint8List bytes = await consolidateHttpClientResponseBytes(
response,
onBytesReceived: (int cumulative, int total) {
chunkEvents.add(ImageChunkEvent(
cumulativeBytesLoaded: cumulative,
expectedTotalBytes: total,
));
},
);
if (bytes.lengthInBytes == 0)
throw Exception('NetworkImage is an empty file: $resolved');
return decode(bytes);
} finally {
chunkEvents.close();
}
}
}
- 第四步：获取到图片Uint8List数据之后就是解码过程。通过DecoderCallback回调方法得到图片原始数据之后交付给全局单例解码器PaintingBinding.instance.instantiateImageCodec，然后由引擎层C++的instantiateImageCodec处理数据返回可被Flutter层渲染展示Image数据。
final ImageStreamCompleter completer = PaintingBinding.instance.imageCache.putIfAbsent(
key,
() => load(key, PaintingBinding.instance.instantiateImageCodec),
onError: handleError,
);
Future<ui.Codec> instantiateImageCodec(Uint8List bytes, {
int cacheWidth,
int cacheHeight,
}) {
assert(cacheWidth == null || cacheWidth > 0);
assert(cacheHeight == null || cacheHeight > 0);
return ui.instantiateImageCodec(
bytes,
targetWidth: cacheWidth,
targetHeight: cacheHeight,
);
}
String _instantiateImageCodec(Uint8List list, _Callback<Codec> callback, _ImageInfo imageInfo, int targetWidth, int targetHeight)
native 'instantiateImageCodec';
- 第五步：引擎层c++解码器具体在codec.cc中，调用了Skia的SkCodec对图片数据做处理。经过解码器内部处理后执行ToDart将ui_codec返回到Dart层。
/// Dart层代码
_String _instantiateImageCodec(Uint8List list, _Callback<Codec> callback, _ImageInfo imageInfo, int targetWidth, int targetHeight)
native 'instantiateImageCodec';
/// c++层代码
static void InstantiateImageCodec(Dart_NativeArguments args) {
UIDartState::ThrowIfUIOperationsProhibited();
Dart_Handle callback_handle = Dart_GetNativeArgument(args, 1);
.......省略部分代码
Dart_Handle image_info_handle = Dart_GetNativeArgument(args, 2);
std::optional<ImageDecoder::ImageInfo> image_info;
/// 图片信息是否为空，不为空做一些处理
if (!Dart_IsNull(image_info_handle)) {
auto image_info_results = ConvertImageInfo(image_info_handle, args);
if (auto value =
std::get_if<ImageDecoder::ImageInfo>(&image_info_results)) {
image_info = *value;
} else if (auto error = std::get_if<std::string>(&image_info_results)) {
Dart_SetReturnValue(args, tonic::ToDart(*error));
return;
}
}
sk_sp<SkData> buffer;
{
/// 处理图片数据
Dart_Handle exception = nullptr;
tonic::Uint8List list =
tonic::DartConverter<tonic::Uint8List>::FromArguments(args, 0,
exception);
if (exception) {
Dart_SetReturnValue(args, exception);
return;
}
/// 图片数据做拷贝
buffer = MakeSkDataWithCopy(list.data(), list.num_elements());
}
if (image_info) {
const auto expected_size =
image_info->row_bytes * image_info->sk_info.height();
if (buffer->size() < expected_size) {
Dart_SetReturnValue(
args, ToDart("Pixel buffer size does not match image size"));
return;
}
}
/// 获取图片目标宽高
const int targetWidth =
tonic::DartConverter<int>::FromDart(Dart_GetNativeArgument(args, 3));
const int targetHeight =
tonic::DartConverter<int>::FromDart(Dart_GetNativeArgument(args, 4));
std::unique_ptr<SkCodec> codec;
bool single_frame;
if (image_info) {
single_frame = true;
} else {
/// 底层解码器使用的是SkCodec解码器，Android底层同样使用的是它。
codec = SkCodec::MakeFromData(buffer);
if (!codec) {
Dart_SetReturnValue(args, ToDart("Could not instantiate image codec."));
return;
}
single_frame = codec->getFrameCount() == 1;
}
/// 解码器同时解码后得出帧数信息，判断图片是否为动图
fml::RefPtr<Codec> ui_codec;
if (single_frame) {
ImageDecoder::ImageDescriptor descriptor;
descriptor.decompressed_image_info = image_info;
if (targetWidth > 0) {
descriptor.target_width = targetWidth;
}
if (targetHeight > 0) {
descriptor.target_height = targetHeight;
}
descriptor.data = std::move(buffer);
ui_codec = fml::MakeRefCounted<SingleFrameCodec>(std::move(descriptor));
} else {
ui_codec = fml::MakeRefCounted<MultiFrameCodec>(std::move(codec));
}
/// 最后将解码器结果返回到Dart层
tonic::DartInvoke(callback_handle, {ToDart(ui_codec)});
}
外接纹理渲染图片
Flutter中有一个叫做Texture组件,该组件只有唯一入参textureId，寥寥无几的几行代码就实现外接纹理着实让人摸不清头脑。在分析外接纹理原理之前先简单了解外接纹理渲染图片功能实现。
Texture组件使用
- Java层通过Channel插件PluginRegistry.Registrar创建Surface、textureId。
/// 插件接口获取texture注册器
TextureRegistry textureRegistry = registrar.textures();
/// 创建Texture实例
TextureRegistry.SurfaceTextureEntry surfaceTextureEntry = textureRegistry.createSurfaceTexture();
long textureId = surfaceTextureEntry.id();
SurfaceTexture surfaceTexture = surfaceTextureEntry.surfaceTexture();
/// 获取图片地址
String url = call.argument("url");
...... 省略图片请求加载过程
/// 创建Surface实例加载surfaceTexture
Surface surface = new Surface(surfaceTexture);
/// 画布绘制bitmap 纹理映射
Canvas canvas = surface.lockCanvas(rect);
canvas.drawBitmap(bitmap, null, rect, null);
bitmap.recycle();
surface.unlockCanvasAndPost(canvas);
/// Dart返回textureId
Map<String, Object> maps = new HashMap<>();
maps.put("textureId", textureId);
result.success(maps);
- Dart层创建MethodChannel，向Native层传递加载图片路径。
static const MethodChannel _channel = const MethodChannel('texture_channel');
/// 原始加载图片接口
static Future<Map> loadTexture({String url}) async {
var args = <String, dynamic>{
"url": url,
};
return await _channel.invokeMethod("loadTexture", args);
}
/// 执行加载
Map _textureResult = await TexturePlugin.loadTexture(
url: _uri.toString(),
width: url.width,
height: url.height,
);
/// 返回Native生成的textureId
int id = _textureResult['textureId'];
/// 实例化texture组件 显示图片
Texture( textureId: id);
代码解析
Dart层
从源码上可以看到Texture会创建渲染对象TextureBox。TextureBox会去绘制TextureLayer，TextureLayer则通过ui.SceneBuilder向Scene添加纹理，最终是调用引擎层SceneBuilder_addTexture方法实现纹理渲染。
- Texture
class Texture extends LeafRenderObjectWidget {
const Texture({
Key key,
@required this.textureId,
}) : assert(textureId != null),
super(key: key);
final int textureId;
@override
TextureBox createRenderObject(BuildContext context) => TextureBox(textureId: textureId);
@override
void updateRenderObject(BuildContext context, TextureBox renderObject) {
renderObject.textureId = textureId;
}
}
- TextureBox
class TextureBox extends RenderBox {
TextureBox({ @required int textureId })
: assert(textureId != null),
_textureId = textureId;
...... 省略代码
@override
void paint(PaintingContext context, Offset offset) {
if (_textureId == null)
return;
context.addLayer(TextureLayer(
rect: Rect.fromLTWH(offset.dx, offset.dy, size.width, size.height),
textureId: _textureId,
));
}
}
- TextureLayer
class TextureLayer extends Layer {
TextureLayer({
@required this.rect,
@required this.textureId,
this.freeze = false,
}) : assert(rect != null),
assert(textureId != null);
......
@override
void addToScene(ui.SceneBuilder builder, [ Offset layerOffset = Offset.zero ]) {
final Rect shiftedRect = layerOffset == Offset.zero ? rect : rect.shift(layerOffset);
builder.addTexture(
textureId,
offset: shiftedRect.topLeft,
width: shiftedRect.width,
height: shiftedRect.height,
freeze: freeze,
);
}
}
- SceneBuilder
class SceneBuilder extends NativeFieldWrapperClass2 {
void addTexture(
int textureId, {
Offset offset = Offset.zero,
double width = 0.0,
double height = 0.0,
bool freeze = false,
}) {
_addTexture(offset.dx, offset.dy, width, height, textureId, freeze);
}
/// SceneBuilder_addTexture对应scene_builder.cc下的SceneBuilder::addTexture方法
void _addTexture(double dx, double dy, double width, double height, int textureId, bool freeze)
native 'SceneBuilder_addTexture';
- scene_builder.cc
void SceneBuilder::addTexture(double dx,
double dy,
double width,
double height,
int64_t textureId,
bool freeze) {
auto layer = std::make_unique<flutter::TextureLayer>(
SkPoint::Make(dx, dy), SkSize::Make(width, height), textureId, freeze);
AddLayer(std::move(layer));
}
- texture_layer.cc
/// 创建纹理层对象
TextureLayer::TextureLayer(const SkPoint& offset,
const SkSize& size,
int64_t texture_id,
bool freeze)
: offset_(offset), size_(size), texture_id_(texture_id), freeze_(freeze) {}
/// 纹理对象绘制方法
void TextureLayer::Paint(PaintContext& context) const {
TRACE_EVENT0("flutter", "TextureLayer::Paint");
/// texture对象绘制时会从GetTexture方法的map中找到纹理对象进行绘制。
std::shared_ptr<Texture> texture =
context.texture_registry.GetTexture(texture_id_);
if (!texture) {
TRACE_EVENT_INSTANT0("flutter", "null texture");
return;
}
texture->Paint(*context.leaf_nodes_canvas, paint_bounds(), freeze_,
context.gr_context);
}
Java层
- FlutterRenderer
public class FlutterRenderer implements TextureRegistry {
public FlutterRenderer(@NonNull FlutterJNI flutterJNI) {
this.flutterJNI = flutterJNI;
this.flutterJNI.addIsDisplayingFlutterUiListener(flutterUiDisplayListener);
}
@Override
public SurfaceTextureEntry createSurfaceTexture() {
/// 创建SurfaceTexture
final SurfaceTexture surfaceTexture = new SurfaceTexture(0);
surfaceTexture.detachFromGLContext();
final SurfaceTextureRegistryEntry entry =
new SurfaceTextureRegistryEntry(nextTextureId.getAndIncrement(), surfaceTexture);
registerTexture(entry.id(), surfaceTexture);
return entry;
}
///向Native JNI层注册
private void registerTexture(long textureId, @NonNull SurfaceTexture surfaceTexture) {
flutterJNI.registerTexture(textureId, surfaceTexture);
}
}
- FlutterJNI
public void registerTexture(long textureId, @NonNull SurfaceTexture surfaceTexture) {
/// 这里需要注意必须在主线程执行否则会报错
ensureRunningOnMainThread();
ensureAttachedToNative();
nativeRegisterTexture(nativePlatformViewId, textureId, surfaceTexture);
}
C++层
- platform_view_android_jni.cc
{
.name = "nativeRegisterTexture",
.signature = "(JJLandroid/graphics/SurfaceTexture;)V",
.fnPtr = reinterpret_cast<void*>(&RegisterTexture),
}
static void RegisterTexture(JNIEnv* env,
jobject jcaller,
jlong shell_holder,
jlong texture_id,
jobject surface_texture) {
ANDROID_SHELL_HOLDER->GetPlatformView()->RegisterExternalTexture(
static_cast<int64_t>(texture_id), //
fml::jni::JavaObjectWeakGlobalRef(env, surface_texture) //
);
}
- platform_view_android.cc
void PlatformViewAndroid::RegisterExternalTexture(
int64_t texture_id,
const fml::jni::JavaObjectWeakGlobalRef& surface_texture) {
RegisterTexture(
std::make_shared<AndroidExternalTextureGL>(texture_id, surface_texture));
}
- texture.cc
// 注册纹理方法
void TextureRegistry::RegisterTexture(std::shared_ptr<Texture> texture) {
if (!texture) {
return;
}
// 内部map保存texture实例
mapping_[texture->Id()] = texture;
}
// 获取到纹理对象，从map中提取
std::shared_ptr<Texture> TextureRegistry::GetTexture(int64_t id) {
auto it = mapping_.find(id);
return it != mapping_.end() ? it->second : nullptr;
}
- android_external_texture_gl.cc
/// 外接纹理对象实例
AndroidExternalTextureGL::AndroidExternalTextureGL(
int64_t id,
const fml::jni::JavaObjectWeakGlobalRef& surfaceTexture)
: Texture(id), surface_texture_(surfaceTexture), transform(SkMatrix::I()) {}
AndroidExternalTextureGL::~AndroidExternalTextureGL() {
if (state_ == AttachmentState::attached) {
glDeleteTextures(1, &texture_name_);
}
}
/// 绘制方法
void AndroidExternalTextureGL::Paint(SkCanvas& canvas,
const SkRect& bounds,
bool freeze,
GrContext* context) {
if (state_ == AttachmentState::detached) {
return;
}
if (state_ == AttachmentState::uninitialized) {
glGenTextures(1, &texture_name_);
Attach(static_cast<jint>(texture_name_));
state_ = AttachmentState::attached;
}
if (!freeze && new_frame_ready_) {
Update();
new_frame_ready_ = false;
}
GrGLTextureInfo textureInfo = {GL_TEXTURE_EXTERNAL_OES, texture_name_,
GL_RGBA8_OES};
GrBackendTexture backendTexture(1, 1, GrMipMapped::kNo, textureInfo);
sk_sp<SkImage> image = SkImage::MakeFromTexture(
canvas.getGrContext(), backendTexture, kTopLeft_GrSurfaceOrigin,
kRGBA_8888_SkColorType, kPremul_SkAlphaType, nullptr);
if (image) {
SkAutoCanvasRestore autoRestore(&canvas, true);
canvas.translate(bounds.x(), bounds.y());
canvas.scale(bounds.width(), bounds.height());
if (!transform.isIdentity()) {
SkMatrix transformAroundCenter(transform);
transformAroundCenter.preTranslate(-0.5, -0.5);
transformAroundCenter.postScale(1, -1);
transformAroundCenter.postTranslate(0.5, 0.5);
canvas.concat(transformAroundCenter);
}
canvas.drawImage(image, 0, 0);
}
}
- ①Java层FlutterRenderer创建SurfaceTexture和textureId。
- ②将urfaceTexture和textureId通过JNI向引擎层注册
- ③向引擎注册过程中通过层层方法最后在texture.cc的TextureRegistry由map以键值对形式缓存实例对象。
- ④将需要显示图片在SurfaceTexture上离屏渲染。
- ⑤Java层创建的textureId通过Channel传递到Dart层作为Texture组件入参。
- ⑥Dart的Texture组件接收textureId入参后向下层组件实例化。
- ⑦在SceneBuilder调用addTexture时执行引擎层创建TextureLayer。
- ⑧最终在texture.cc中TextureRegistry的map根据TextureId获取SurfaceTexture实例。
Image VS Texture
Image组件和Texture组件实质上都是对RenderBox的实现，已知Flutter渲染树实际上就是RenderObject合并为Layer最终通过Flutter引擎进行绘制上屏显示页面内容。两者区别在于实现RenderObject的paint有所不同，Image的实现将ui.Image内容绘制在ui.Canvas上，Texture是将TextureLayer添加到ui.Scene中。不同之处就在于Image做了绘制操作，Texture是做添加操作，纹理方案的图片加载和绘制完全由原生平台完成。
