tencent cloud

PrevNext

Feedback

search by keyword

Recent Pages

Documentation
Tencent Real-Time Communication
    Documentation
    Download PDF

    Flutter

    Last updated: 2024-04-03 17:23:11
    Download PDF
      This document mainly introduces the method of integrating beauty effects in TUICallKit.
      To complete custom beauty processing in Flutter, it needs to be done through TRTC custom video rendering. Due to Flutter's characteristics of not being good at handling a large amount of real-time data transmission, the part involving TRTC custom video rendering needs to be completed in the Native part. The specific plan is as follows:
      
      
      The access plan is divided into 3 steps:
      Step 1: Enable/disable TRTC custom rendering logic through the MethodChannel.
      Step 2: Use the beauty processing module in TRTC's custom rendering processing logic onProcessVideoFrame() to process the original video frame.
      Step 3: The customer's beauty processing module also needs to set the current beauty parameters through the interface in Dart. Customers can set beauty parameters through the MethodChannel method. This part can be customized by the customer according to their needs and the beauty they use.

      Integrate third-party beauty effects

      Step 1: Implement the start/end beauty control interface from Dart layer to Native

      Implement Dart layer interface: 
       final channel = MethodChannel('TUICallKitCustomBeauty');
        
       void enableTUICallKitCustomBeauty() async {
          await channel.invokeMethod('enableTUICallKitCustomBeauty');   
       }
      
       void disableTUICallKitCustomBeauty() async {
          await channel.invokeMethod('disableTUICallKitCustomBeauty');
       }
      Implement the corresponding Native layer interface:
      java
      swift
      public class MainActivity extends FlutterActivity {
          private static final String channelName = "TUICallKitCustomBeauty";
          
          private MethodChannel channel;
          
          @Override
          public void configureFlutterEngine(@NonNull FlutterEngine flutterEngine) {
              super.configureFlutterEngine(flutterEngine);
              
              channel = new MethodChannel(flutterEngine.getDartExecutor().getBinaryMessenger(), channelName);
              channel.setMethodCallHandler(((call, result) -> {
                  switch (call.method) {
                      case "enableTUICallKitCustomBeauty":
                          enableTUICallKitCustomBeauty();
                          break;
                      case "disableTUICallKitCustomBeauty":
                          disableTUICallKitCustomBeauty();
                          break;
                      default:
                          break;
                  }
                  result.success("");
              }));
          }
          
          public void enableTUICallKitCustomBeauty() {
          
          }
          
          public void disableTUICallKitCustomBeauty() {
          
          }
      }
      @UIApplicationMain
      @objc class AppDelegate: FlutterAppDelegate {
          var channel: FlutterMethodChannel?
          
          override func application(_ application: UIApplication,
                                    didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]?) -> Bool {
              GeneratedPluginRegistrant.register(with: self)
         
              guard let controller = window?.rootViewController as? FlutterViewController else {
                  fatalError("Invalid root view controller")
              }
              channel = FlutterMethodChannel(name: "TUICallKitCustomBeauty", binaryMessenger: controller.binaryMessenger)
              channel?.setMethodCallHandler({ [weak self] call, result in
                  guard let self = self else { return }
                  switch (call.method) {
                  case "enableTUICallKitCustomBeauty":
                      self.enableTUICallKitCustomBeauty()
                      break
                  case "disableTUICallKitCustomBeauty":
                      self.disableTUICallKitCustomBeauty()
                      break
                  default:
                      break
                  }
              })    
              result(nil)   
              return super.application(application, didFinishLaunchingWithOptions: launchOptions)
          }
          
          func enableTUICallKitCustomBeauty() {
          
          }
          
          func disableTUICallKitCustomBeauty() {
          
          }
      }

      Step 2: Complete beauty processing in Native TRTC custom rendering logic

      Note：
      Android needs to rely on LiteAVSDK_Professional first when accessing beauty. Add the following dependencies in the app/build.gradle of the Android project:
      dependencies{ api "com.tencent.liteav:LiteAVSDK_Professional:latest.release" }
      java
      swift
      void enableTUICallKitCustomBeauty() {
          TRTCCloud.sharedInstance(getApplicationContext()).
                      setLocalVideoProcessListener(TRTC_VIDEO_PIXEL_FORMAT_Texture_2D,
                              TRTC_VIDEO_BUFFER_TYPE_TEXTURE, new VideoFrameListerer());
      
      }
      
      void disableTUICallKitCustomBeauty() {
          TRTCCloud.sharedInstance(getApplicationContext()).
                      setLocalVideoProcessListener(TRTC_VIDEO_PIXEL_FORMAT_Texture_2D,
                              TRTC_VIDEO_BUFFER_TYPE_TEXTURE, null);
      }
      
      class VideoFrameListerer implements TRTCCloudListener.TRTCVideoFrameListener {
    private XXXBeautyModel  mBeautyModel = XXXBeautyModel.sharedInstance(); 
         
    @Override
    public int onProcessVideoFrame(TRTCCloudDef.TRTCVideoFrame trtcVideoFrame,
                                   TRTCCloudDef.TRTCVideoFrame trtcVideoFrame1) {
        // Beauty processing logic
              mBeautyModel.process(trtcVideoFrame, trtcVideoFrame1);
              ……
              
        return 0;
    }

    @Override
    public void onGLContextCreated() {
    }

    @Override
    public void onGLContextDestory() {
    }
}
      let videoFrameListener: TRTCVideoFrameListener = TRTCVideoFrameListener()
      
      func enableTUICallKitCustomBeauty() {
          TRTCCloud.sharedInstance().setLocalVideoProcessDelegete(videoFrameListener, pixelFormat: ._Texture_2D, bufferType: .texture)
      }
      
      func disableTUICallKitCustomBeauty() {
          TRTCCloud.sharedInstance().setLocalVideoProcessDelegete(nil, pixelFormat: ._Texture_2D, bufferType: .texture)
      }
      
      class TRTCVideoFrameListener: NSObject, TRTCVideoFrameDelegate {
          let bueutyModel = XXXXBeautyModel.shareIntance()
          func onProcessVideoFrame(_ srcFrame: TRTCVideoFrame, dstFrame: TRTCVideoFrame) -> UInt32 {
              // Beauty processing logic
              bueutyModel.onProcessVideoFrame(srcFrame, dstFrame)
              ……
              
              return 0
          }
      }

      Step 3: Customer-defined third-party beauty parameter control logic

      In this part, customers can set beauty parameters according to their needs and the specific beauty module they use, referring to the implementation in step 1. The specific implementation depends on the specific usage.

      Integrating Tencent Beauty Effects

      The integration method of Tencent Beauty Effects also follows the above method. Now, taking Tencent Beauty Effects as an example, we will introduce the integration method in detail:

      Step 1: Beauty resource download and integration

      1. Download the SDK according to the package you purchased.
      2. Add files to your project:
      Android
      iOS
      1. Find the build.gradle file under the app module and add the maven reference address for your corresponding package. For example, if you choose the S1-04 package, add the following:
      dependencies {
         implementation 'com.tencent.mediacloud:TencentEffect_S1-04:latest.release'
      }
      For the maven addresses corresponding to each package, please refer to the documentation.
      2. Find the src/main/assets folder under the app module. If it doesn't exist, create it. Check if there is a MotionRes folder in the downloaded SDK package. If so, copy this folder to the ../src/main/assets directory.
      3. Find the AndroidManifest.xml file under the app module and add the following tag in the application form
       <uses-native-library
              android:name="libOpenCL.so"
              android:required="true" />
              //The "true" here means that if this library is not present, the application will not run properly. The system does not allow the installation of applications on devices without this library.
               // "false" means that the application can use this library (if it exists) but is specifically designed to run without it (if necessary). The system allows the installation of applications even if this library does not exist. If you use "false", you need to take responsibility for properly handling the absence of the library.
      Add as shown in the following figure:
      
      
      4. Obfuscation configuration
      If you enable compile optimization when building a release package (set minifyEnabled to true), some code that is not called in the java layer will be cut off, and this code may be called by the native layer, causing a no xxx method exception.
      If you enable such compile optimization, you need to add these keep rules to prevent xmagic code from being cut off:
      -keep class com.tencent.xmagic.** { *;}
      -keep class org.light.** { *;}
      -keep class org.libpag.** { *;}
      -keep class org.extra.** { *;}
      -keep class com.gyailib.**{ *;}
      -keep class com.tencent.cloud.iai.lib.** { *;}
      -keep class com.tencent.beacon.** { *;}
      -keep class com.tencent.qimei.** { *;}
      -keep class androidx.exifinterface.** { *;}
      1. Add beauty resources to your project. After adding, it will be shown as in the following figure (the types of resources you have may not be exactly the same as in the figure):
      
      
      2. In the demo, copy the 4 classes in demo/lib/producer: BeautyDataManager, BeautyPropertyProducer, BeautyPropertyProducerAndroid, and BeautyPropertyProducerIOS to your own Flutter project. These 4 classes are used to configure beauty resources and display beauty types on the beauty panel.

      Step 2: Reference the Flutter version SDK

      GitHub reference: Add the following reference to the project's pubspec.yaml file:
       tencent_effect_flutter:
         git:
           url: https://github.com/TencentCloud/tencenteffect-sdk-flutter
      Local reference: Download the latest version of tencent_effect_flutter from tencent_effect_flutter, and then add the folders android, ios, lib, and the files pubspec.yaml, tencent_effect_flutter.iml to the project directory. Then, add the following reference to the project's pubspec.yaml file (refer to the demo):
      tencent_effect_flutter:
          path: ../
      tencent_effect_flutter provides only a bridge, and the default version of the internally dependent XMagic is the latest. The real beauty effect is achieved by XMagic.
      If you want to use the latest version of the beauty SDK, you can upgrade the SDK through the following steps:
      Android
      iOS
      Execute the command flutter pub upgrade in the project directory, or click "Pub upgrade" in the upper right corner of the subspec.yaml page.
      Execute the command flutter pub upgrade in the project directory, and then execute the command pod update in the iOS directory.

      Step 3: Implement the Dart layer to Native beauty control interface start/end

      This section can refer to Accessing third-party beauty effects/Step 1, and is not repeated here.

      Step 4: Complete the beauty processing in the TRTC custom rendering logic of Native

      Android
      iOS
      Android needs to rely on LiteAVSDK_Professional first when accessing beauty. Add the following dependencies in the app/build.gradle of the Android project:
      dependencies{ api "com.tencent.liteav:LiteAVSDK_Professional:latest.release" }
      void enableTUICallKitCustomBeauty() {
          TRTCCloud.sharedInstance(getApplicationContext()).
                      setLocalVideoProcessListener(TRTC_VIDEO_PIXEL_FORMAT_Texture_2D,
                              TRTC_VIDEO_BUFFER_TYPE_TEXTURE, new VideoFrameListerer());
      
      }
      
      void disableTUICallKitCustomBeauty() {
          TRTCCloud.sharedInstance(getApplicationContext()).
                      setLocalVideoProcessListener(TRTC_VIDEO_PIXEL_FORMAT_Texture_2D,
                              TRTC_VIDEO_BUFFER_TYPE_TEXTURE, null);
      }
      
      class VideoFrameListerer implements TRTCCloudListener.TRTCVideoFrameListener {
          private XXXBeautyModel  mBeautyModel = XXXBeautyModel.sharedInstance(); 
         
          @Override
          public int onProcessVideoFrame(TRTCCloudDef.TRTCVideoFrame trtcVideoFrame,
                                         TRTCCloudDef.TRTCVideoFrame trtcVideoFrame1) {
             trtcVideoFrame1.texture.textureId = XmagicApiManager.getInstance()
        .process(trtcVideoFrame.texture.textureId, trtcVideoFrame.width, trtcVideoFrame.height);
              return 0;
          }
      
          @Override
          public void onGLContextCreated() {
          }
      
          @Override
          public void onGLContextDestory() {
          }
      }
      import TXLiteAVSDK_Professional
      import tencent_effect_flutter
      
      let videoFrameListener: TRTCVideoFrameListener = TRTCVideoFrameListener()
      
      func enableTUICallKitCustomBeauty() {
          TRTCCloud.sharedInstance().setLocalVideoProcessDelegete(videoFrameListener, pixelFormat: ._Texture_2D, bufferType: .texture)
      }
      
      func disableTUICallKitCustomBeauty() {
          TRTCCloud.sharedInstance().setLocalVideoProcessDelegete(nil, pixelFormat: ._Texture_2D, bufferType: .texture)
      }
      
      class TRTCVideoFrameListener: NSObject, TRTCVideoFrameDelegate {
          func onProcessVideoFrame(_ srcFrame: TRTCVideoFrame, dstFrame: TRTCVideoFrame) -> UInt32 {
              dstFrame.textureId = GLuint(XmagicApiManager.shareSingleton().getTextureId(ConvertBeautyFrame.convertTRTCVideoFrame(trtcVideoFrame: srcFrame)))
              return 0
          }
      }
      
      
      public class ConvertBeautyFrame: NSObject {
          public static func convertToTRTCPixelFormat(beautyPixelFormat: ITXCustomBeautyPixelFormat) -> TRTCVideoPixelFormat {
              switch beautyPixelFormat {
              case .Unknown:
                  return ._Unknown
              case .I420:
                  return ._I420
              case .Texture2D:
                  return ._Texture_2D
              case .BGRA:
                  return ._32BGRA
              case .NV12:
                  return ._NV12
              }
          }
      
          public static func convertTRTCVideoFrame(trtcVideoFrame: TRTCVideoFrame) -> ITXCustomBeautyVideoFrame {
              let beautyVideoFrame = ITXCustomBeautyVideoFrame()
              beautyVideoFrame.data = trtcVideoFrame.data
              beautyVideoFrame.pixelBuffer = trtcVideoFrame.pixelBuffer
              beautyVideoFrame.width = UInt(trtcVideoFrame.width)
              beautyVideoFrame.height = UInt(trtcVideoFrame.height)
              beautyVideoFrame.textureId = trtcVideoFrame.textureId
              switch trtcVideoFrame.rotation {
              case ._0:
                  beautyVideoFrame.rotation = .rotation_0
              case ._90:
                  beautyVideoFrame.rotation = .rotation_90
              case ._180:
                  beautyVideoFrame.rotation = .rotation_180
              case ._270:
                  beautyVideoFrame.rotation = .rotation_270
              default:
                  beautyVideoFrame.rotation = .rotation_0
              }
              switch trtcVideoFrame.pixelFormat {
              case ._Unknown:
                  beautyVideoFrame.pixelFormat = .Unknown
              case ._I420:
                  beautyVideoFrame.pixelFormat = .I420
              case ._Texture_2D:
                  beautyVideoFrame.pixelFormat = .Texture2D
              case ._32BGRA:
                  beautyVideoFrame.pixelFormat = .BGRA
              case ._NV12:
                  beautyVideoFrame.pixelFormat = .NV12
              default:
                  beautyVideoFrame.pixelFormat = .Unknown
              }
              beautyVideoFrame.bufferType = ITXCustomBeautyBufferType(rawValue: trtcVideoFrame.bufferType.rawValue) ?? .Unknown
              beautyVideoFrame.timestamp = trtcVideoFrame.timestamp
              return beautyVideoFrame
          }
      }
      

      Step 5: Enable beauty and set beauty parameters

      After completing the above configuration, you can enable/disable beauty through enableTUICallKitCustomBeauty()/disableTUICallKitCustomBeauty(). You can set beauty parameters through the Tencent beauty effects Flutter interface.
      
      Contact Us

      Contact our sales team or business advisors to help your business.

      Contact Us
      Technical Support

      Open a ticket if you're looking for further assistance. Our Ticket is 7x24 avaliable.

      Submit a Ticket
      7x24 Phone Support
      Copyright © 2013-2024 Tencent Cloud. All Rights Reserved.
      Privacy PolicyLegalCookie Policy