motionlib: add suite of RenderEffect-based MotionEffects

gradle/libs.versions.toml

@@ -34,6 +34,7 @@ runtimeVersion = "1.10.6"
 crashlyticsVersion = "3.0.7"
 googleServicesVersion = "4.4.4"
 media3Version = "1.10.0"
+mlkitSubjectSegmentationVersion = "16.0.0-beta1"
 
 [libraries]
 androidx-constraintlayout = { module = "androidx.constraintlayout:constraintlayout", version.ref = "constraintlayoutVersion" }
@@ -84,6 +85,7 @@ androidx-ui-test-junit4 = { group = "androidx.compose.ui", name = "ui-test-junit
 androidx-material3 = { group = "androidx.compose.material3", name = "material3" }
 androidx-navigation-compose = { group = "androidx.navigation", name = "navigation-compose", version.ref = "navigationComposeVersion" }
 androidx-compose-runtime = { group = "androidx.compose.runtime", name = "runtime", version.ref = "runtimeVersion" }
+mlkit-subject-segmentation = { module = "com.google.android.gms:play-services-mlkit-subject-segmentation", version.ref = "mlkitSubjectSegmentationVersion" }
 compose-material-icons-extended = { module = "androidx.compose.material:material-icons-extended" }
 
 supabase-bom = { module = "io.github.jan-tennert.supabase:bom", version.ref = "supabaseBomVersion" }

modules/lyrics-maker/src/main/java/com/tejpratapsingh/lyricsmaker/presentation/templates/LyricsTemplateRegistry.kt

@@ -14,6 +14,7 @@ object LyricsTemplateRegistry {
             GradientLyricsTemplate,
             RainbowLyricsTemplate,
             AccentLyricsTemplate,
+            VintageLyricsTemplate,
         )
 
     fun getTemplate(name: String?): MotionTemplate = templates.find { it.name == name } ?: PopupLyricsTemplate

modules/lyrics-maker/src/main/java/com/tejpratapsingh/lyricsmaker/presentation/templates/PopupLyricsTemplate.kt

@@ -12,6 +12,7 @@ import com.tejpratapsingh.motionlib.templates.extensions.motionImageView
 import com.tejpratapsingh.motionlib.templates.extensions.popUpTextView
 import com.tejpratapsingh.motionlib.templates.extensions.translucentMotionView
 import com.tejpratapsingh.motionlib.templates.model.MotionTemplate
+import com.tejpratapsingh.motionlib.ui.effects.BlurEffect
 
 val PopupLyricsTemplate: MotionTemplate =
     motionTemplate("Multi Lyrics Template") {
@@ -41,6 +42,13 @@ val PopupLyricsTemplate: MotionTemplate =
                     alpha = 0.4f,
                     startFrame = lyrics.first().frame,
                     endFrame = lyrics.last().frame,
+                    effects =
+                        listOf(
+                            BlurEffect(
+                                lyrics.first().frame,
+                                lyrics.last().frame,
+                            ),
+                        ),
                 )
 
                 lyrics.zipWithNext().forEach { (current, next) ->
@@ -86,6 +94,13 @@ val PopupLyricsTemplate: MotionTemplate =
                     alpha = 0.4f,
                     startFrame = previewLyrics.first().frame,
                     endFrame = previewLyrics.last().frame,
+                    effects =
+                        listOf(
+                            BlurEffect(
+                                previewLyrics.first().frame,
+                                previewLyrics.last().frame,
+                            ),
+                        ),
                 )
 
                 previewLyrics.zipWithNext().forEach { (current, next) ->

modules/lyrics-maker/src/main/java/com/tejpratapsingh/lyricsmaker/presentation/templates/VintageLyricsTemplate.kt

@@ -0,0 +1,121 @@
+package com.tejpratapsingh.lyricsmaker.presentation.templates
+
+import android.view.Gravity
+import androidx.appcompat.widget.AppCompatTextView
+import androidx.core.net.toUri
+import com.tejpratapsingh.lyricsmaker.data.lrc.SyncedLyricFrame
+import com.tejpratapsingh.motionlib.core.MotionTextVariant
+import com.tejpratapsingh.motionlib.core.motion.transitions.SlideDirection
+import com.tejpratapsingh.motionlib.core.motion.transitions.SlideTransition
+import com.tejpratapsingh.motionlib.templates.dsl.motionTemplate
+import com.tejpratapsingh.motionlib.templates.extensions.motionImageView
+import com.tejpratapsingh.motionlib.templates.extensions.translucentMotionView
+import com.tejpratapsingh.motionlib.templates.extensions.wordWriterTextView
+import com.tejpratapsingh.motionlib.templates.model.MotionTemplate
+import com.tejpratapsingh.motionlib.ui.effects.VibrateEffect
+import com.tejpratapsingh.motionlib.ui.effects.VintageEffect
+
+val VintageLyricsTemplate: MotionTemplate =
+    motionTemplate("Vintage Lyrics Template") {
+        parameters {
+            string("songName")
+            string("image", defaultValue = null)
+        }
+
+        content {
+            val image = data.getString("image")
+            val lyrics = data.get<List<SyncedLyricFrame>>("lyrics") ?: emptyList()
+
+            if (lyrics.isNotEmpty()) {
+                val startFrame = lyrics.first().frame
+                val endFrame = lyrics.last().frame
+
+                image?.let {
+                    motionImageView(
+                        startFrame = startFrame,
+                        endFrame = endFrame,
+                        imageUri = image.toUri(),
+                        effects =
+                            listOf(
+                                VibrateEffect(startFrame, endFrame, amplitude = 10f, frequency = 0.05f),
+                                VintageEffect(startFrame, endFrame, fromIntensity = 0.8f, toIntensity = 1.0f),
+                            ),
+                    )
+                }
+
+                translucentMotionView(
+                    color = "#000000",
+                    alpha = 0.3f,
+                    startFrame = startFrame,
+                    endFrame = endFrame,
+                )
+
+                lyrics.zipWithNext().forEach { (current, next) ->
+                    wordWriterTextView(
+                        text = current.text,
+                        startFrame = current.frame,
+                        endFrame = next.frame,
+                        textSizeVariant = MotionTextVariant.H1,
+                        textColor = "#FFFFFF",
+                        writingSpeed = 1.0f,
+                        textView =
+                            AppCompatTextView(context).apply {
+                                setPadding(32, 32, 32, 32)
+                                textAlignment = AppCompatTextView.TEXT_ALIGNMENT_CENTER
+                                gravity = Gravity.CENTER
+                            },
+                    )
+                    transition(SlideTransition(SlideDirection.RIGHT_TO_LEFT), duration = 15)
+                }
+            }
+        }
+
+        preview {
+            val image = data.getString("image")
+            val lyrics = data.get<List<SyncedLyricFrame>>("lyrics") ?: emptyList()
+
+            if (lyrics.isNotEmpty()) {
+                val previewLyrics = lyrics.take(3)
+                val startFrame = previewLyrics.first().frame
+                val endFrame = previewLyrics.last().frame
+
+                image?.let {
+                    motionImageView(
+                        startFrame = startFrame,
+                        endFrame = endFrame,
+                        imageUri = image.toUri(),
+                        effects =
+                            listOf(
+                                VibrateEffect(startFrame, endFrame, amplitude = 10f, frequency = 0.05f),
+                                VintageEffect(startFrame, endFrame, fromIntensity = 0.8f, toIntensity = 1.0f),
+                            ),
+                    )
+                }
+
+                translucentMotionView(
+                    color = "#000000",
+                    alpha = 0.3f,
+                    startFrame = startFrame,
+                    endFrame = endFrame,
+                )
+
+                previewLyrics.zipWithNext().forEach { (current, next) ->
+                    wordWriterTextView(
+                        text = current.text,
+                        startFrame = current.frame,
+                        endFrame = next.frame,
+                        textSizeVariant = MotionTextVariant.H1,
+                        textColor = "#FFFFFF",
+                        writingSpeed = 1.0f,
+                        textView =
+                            AppCompatTextView(context).apply {
+                                setPadding(32, 32, 32, 32)
+                                textAlignment = AppCompatTextView.TEXT_ALIGNMENT_CENTER
+                                gravity = Gravity.CENTER
+                            },
+                    )
+                    transition(SlideTransition(SlideDirection.RIGHT_TO_LEFT), duration = 15)
+                }
+            }
+        }
+    }

modules/ml-kit-ext/build.gradle

@@ -0,0 +1,55 @@
+plugins {
+    alias(libs.plugins.android.library)
+    id 'maven-publish'
+}
+
+android {
+    compileSdk 36
+
+    defaultConfig {
+        minSdk 25
+
+        testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
+        consumerProguardFiles "consumer-rules.pro"
+    }
+
+    buildTypes {
+        release {
+            minifyEnabled false
+            proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
+        }
+    }
+    compileOptions {
+        sourceCompatibility JavaVersion.VERSION_11
+        targetCompatibility JavaVersion.VERSION_11
+    }
+
+    namespace 'com.tejpratapsingh.motionlib.mlkit'
+    lint {
+        targetSdk 36
+    }
+    testOptions {
+        targetSdk 36
+    }
+    publishing {
+        singleVariant("release")
+    }
+}
+
+afterEvaluate {
+    publishing {
+        publications {
+            release(MavenPublication) {
+                from components.release
+            }
+        }
+    }
+}
+
+dependencies {
+    api project(path: ':modules:core')
+    implementation libs.androidx.appcompat
+    implementation libs.mlkit.subject.segmentation
+
+    testImplementation libs.junit
+}

modules/ml-kit-ext/src/main/java/com/tejpratapsingh/motionlib/mlkit/MLKitImageProcessor.kt

@@ -0,0 +1,16 @@
+package com.tejpratapsingh.motionlib.mlkit
+
+import com.tejpratapsingh.motionlib.core.MotionPlugin
+import com.tejpratapsingh.motionlib.mlkit.plugins.SubjectSegmentationPlugin
+
+/**
+ * Entry point for ML Kit based image processing plugins.
+ */
+object MLKitImageProcessor {
+    /**
+     * Plugin for background removal using ML Kit Subject Segmentation.
+     */
+    val subjectSegmentationPlugin: MotionPlugin by lazy {
+        SubjectSegmentationPlugin()
+    }
+}

modules/ml-kit-ext/src/main/java/com/tejpratapsingh/motionlib/mlkit/effects/SubjectSegmentationEffect.kt

@@ -0,0 +1,117 @@
+package com.tejpratapsingh.motionlib.mlkit.effects
+
+import android.graphics.Bitmap
+import android.graphics.BitmapShader
+import android.graphics.Color
+import android.graphics.RenderEffect
+import android.graphics.RuntimeShader
+import android.graphics.Shader
+import android.os.Build
+import android.view.View
+import androidx.core.graphics.createBitmap
+import com.google.android.gms.tasks.Tasks
+import com.google.mlkit.vision.common.InputImage
+import com.google.mlkit.vision.segmentation.subject.SubjectSegmentation
+import com.google.mlkit.vision.segmentation.subject.SubjectSegmenterOptions
+import com.tejpratapsingh.motionlib.core.MotionEffect
+import com.tejpratapsingh.motionlib.core.MotionView
+import java.nio.FloatBuffer
+
+/**
+ * A [MotionEffect] that uses ML Kit Subject Segmentation to remove background.
+ * Works only on Android T (API 33) and above due to [RenderEffect] and [RuntimeShader].
+ */
+class SubjectSegmentationEffect(
+    override val startFrame: Int,
+    override val endFrame: Int,
+) : MotionEffect {
+    override lateinit var motionView: MotionView
+
+    private val options =
+        SubjectSegmenterOptions
+            .Builder()
+            .enableForegroundConfidenceMask()
+            .build()
+
+    private val segmenter = SubjectSegmentation.getClient(options)
+
+    private val maskCache = mutableMapOf<Int, Bitmap>()
+
+    @Suppress("ktlint:standard:property-naming")
+    private val MASK_SHADER =
+        """
+        uniform shader content;
+        uniform shader mask;
+
+        half4 main(float2 fragCoord) {
+            half4 color = content.eval(fragCoord);
+            half4 maskColor = mask.eval(fragCoord);
+            // Use alpha channel of the mask for transparency
+            return color * maskColor.a;
+        }
+        """.trimIndent()
+
+    override fun forFrame(frame: Int): MotionView {
+        if (motionView !is View) return motionView
+        val view = motionView as View
+
+        if (Build.VERSION.SDK_INT < Build.VERSION_CODES.TIRAMISU) return motionView
+
+        if (frame !in startFrame..endFrame) {
+            // If we are past the end frame, clear the effect
+            if (frame > endFrame) {
+                view.setRenderEffect(null)
+            }
+            return motionView
+        }
+
+        val maskBitmap =
+            maskCache[frame] ?: run {
+                val bitmap = motionView.getViewBitmap()
+                val image = InputImage.fromBitmap(bitmap, 0)
+                try {
+                    // This is synchronous and can be slow.
+                    // Ideally, this should be pre-processed or run on a background thread.
+                    val result = Tasks.await(segmenter.process(image))
+                    val mask = result.foregroundConfidenceMask // This is a FloatBuffer
+                    if (mask != null) {
+                        val createdMask = createBitmapFromMask(mask, bitmap.width, bitmap.height)
+                        maskCache[frame] = createdMask
+                        createdMask
+                    } else {
+                        null
+                    }
+                } catch (_: Exception) {
+                    null
+                }
+            }
+
+        if (maskBitmap != null) {
+            val shader = RuntimeShader(MASK_SHADER)
+            val maskShader = BitmapShader(maskBitmap, Shader.TileMode.CLAMP, Shader.TileMode.CLAMP)
+            shader.setInputShader("mask", maskShader)
+
+            view.setRenderEffect(RenderEffect.createRuntimeShaderEffect(shader, "content"))
+        }
+
+        return motionView
+    }
+
+    private fun createBitmapFromMask(
+        mask: FloatBuffer,
+        width: Int,
+        height: Int,
+    ): Bitmap {
+        val bitmap = createBitmap(width, height)
+        mask.rewind()
+        val pixels = IntArray(width * height)
+        for (i in 0 until width * height) {
+            val confidence = if (mask.hasRemaining()) mask.get() else 0.0f
+            val alpha = (confidence * 255).toInt()
+            // Set alpha for the mask bitmap, color doesn't strictly matter if we only use alpha in shader
+            pixels[i] = Color.argb(alpha, 255, 255, 255)
+        }
+        bitmap.setPixels(pixels, 0, width, 0, 0, width, height)
+        return bitmap
+    }
+}