From 77d92657ad19f0ca4db232d5c991bbe2b81e18da Mon Sep 17 00:00:00 2001 From: Amlan Roy Date: Sun, 26 Dec 2021 15:50:28 +0530 Subject: [PATCH 1/2] Changed project Structure to follow Factory Class Design Pattern The image analyser calls DrawExercise Custom View, which uses factory class to create dumbell curls exercise passing deviceWidth/2,deviceLength/2 to ImageAnalysisBuilder scalex and scale y hardcoded to 2 hardcoded x and y offset In dumbell curls.drawing,Calling init function everytime at the beginning, so that it checks for elbow position and hence right side or left side, every time Reps reset to 0 after 3 sets Sets increase by one after 12 reps(on 13th rep) Added a timer, which starts on button click It displays time left in text view After finish, button is removed, textView is removed and Stop button is made visible if isActive and not finished, then we do processing on the detected pose, else nothing onClick of stop button, isActive false, isFinished true,remove all the drawing if any --- .../example/mltestapplication/MainActivity.kt | 117 +++++++++++++----- .../com/example/mltestapplication/new/Draw.kt | 56 ++++++--- .../mltestapplication/new/DrawExercise.kt | 8 +- .../mltestapplication/new/DumbellCurls.kt | 61 +++++++-- .../example/mltestapplication/new/Exercise.kt | 3 +- .../example/mltestapplication/new/Factory.kt | 4 +- .../app/src/main/res/layout/activity_main.xml | 30 ++++- 7 files changed, 208 insertions(+), 71 deletions(-) diff --git a/MLTestApplication/app/src/main/java/com/example/mltestapplication/MainActivity.kt b/MLTestApplication/app/src/main/java/com/example/mltestapplication/MainActivity.kt index 3dfd934..5a50833 100644 --- a/MLTestApplication/app/src/main/java/com/example/mltestapplication/MainActivity.kt +++ b/MLTestApplication/app/src/main/java/com/example/mltestapplication/MainActivity.kt @@ -1,13 +1,20 @@ package com.example.mltestapplication -import androidx.appcompat.app.AppCompatActivity -import android.os.Bundle import android.annotation.SuppressLint +import android.os.Bundle +import android.os.CountDownTimer import android.util.Log import android.util.Size -import androidx.core.content.ContextCompat +import android.view.Gravity +import android.view.View +import android.view.ViewGroup +import android.widget.Button +import android.widget.LinearLayout +import android.widget.TextView +import androidx.appcompat.app.AppCompatActivity import androidx.camera.core.* import androidx.camera.lifecycle.ProcessCameraProvider +import androidx.core.content.ContextCompat import androidx.databinding.DataBindingUtil import androidx.lifecycle.LifecycleOwner import com.example.mltestapplication.databinding.ActivityMainBinding @@ -20,22 +27,29 @@ import com.google.mlkit.vision.pose.Pose import com.google.mlkit.vision.pose.PoseDetection import com.google.mlkit.vision.pose.PoseDetector import com.google.mlkit.vision.pose.defaults.PoseDetectorOptions +import kotlinx.android.synthetic.main.activity_main.* + class MainActivity : AppCompatActivity(){ private lateinit var binding: ActivityMainBinding private lateinit var poseDetector : PoseDetector private lateinit var cameraProviderFuture: ListenableFuture + private lateinit var countdown_timer: CountDownTimer + private lateinit var start_button: Button + private var isActive : Boolean = false +// private lateinit var textView : TextView + private var isFinished : Boolean = false + override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) binding = DataBindingUtil.setContentView(this,R.layout.activity_main) - //getSupportActionBar()?.hide() + supportActionBar?.hide() cameraProviderFuture = ProcessCameraProvider.getInstance(this) cameraProviderFuture.addListener({ - Log.d("CALLED", "Called inside cameraProviderFuture.addListener") val cameraProvider = cameraProviderFuture.get() bindPreview(cameraProvider = cameraProvider) }, ContextCompat.getMainExecutor(this)) @@ -49,16 +63,48 @@ class MainActivity : AppCompatActivity(){ .build() - poseDetector = PoseDetection.getClient(poseDetectorOptions) + start_button = findViewById(R.id.id_start_button) as Button + start_button.setOnClickListener{ + startTimer(5) + } + + id_stop_button.setOnClickListener{ + isActive = false + isFinished = true + if (binding.parentLayout.childCount > 2) binding.parentLayout.removeViews( + 2, + binding.parentLayout.childCount - 2 + ) + } + countdown.visibility = View.INVISIBLE + id_stop_button.visibility = View.INVISIBLE + + } + private fun startTimer(time_in_seconds: Long) { + binding.parentLayout.removeView(start_button) + countdown.visibility = View.VISIBLE + countdown_timer = object : CountDownTimer(time_in_seconds*1000, 1000) { + override fun onFinish() { + isActive = true + binding.parentLayout.removeView(countdown) + id_stop_button.visibility = View.VISIBLE + } + + override fun onTick(timeLeft: Long) { + countdown.text = (timeLeft/1000).toString() + } + } + countdown_timer.start() + + } @SuppressLint("UnsafeOptInUsageError") private fun bindPreview(cameraProvider: ProcessCameraProvider){ - Log.d("CALLED", "Called Just inside Bind preview") val preview = Preview.Builder().build() - +// when true, analyses, displays and updates sets and reps of exercise //defining the camera, the prerequisites before opening the camera val cameraSelector = CameraSelector.Builder() @@ -69,35 +115,42 @@ class MainActivity : AppCompatActivity(){ val imageAnalysis = ImageAnalysis.Builder() - .setTargetResolution(Size(1280,720)) + .setTargetResolution(Size(resources.displayMetrics.heightPixels / 2,resources.displayMetrics.widthPixels / 2 )) .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) .build() - imageAnalysis.setAnalyzer(ContextCompat.getMainExecutor(this),{imageProxy -> - Log.d("CALLED", "Called inside imageAnalysis.setAnalyzer") - val rotationDegrees = imageProxy.imageInfo.rotationDegrees - val image = imageProxy.image - if(image != null){ - val processImage = InputImage.fromMediaImage(image, rotationDegrees) - var element : DrawExercise - val result : Task = poseDetector - .process(processImage) - .addOnSuccessListener { pose -> -// val allPoseLandmarks = pose.allPoseLandmarks - Log.d("CALLED", "Called inside poseDetector.onSuccessListener") - if (binding.parentLayout.childCount > 1) binding.parentLayout.removeViews( 1, - binding.parentLayout.childCount - 1) - element =DrawExercise(ExerciseType.DumbellCurl_Type,context=this, pose) - binding.parentLayout.addView(element) - imageProxy.close() - }.addOnFailureListener { - Log.v("MainActivity", "Error - ${it.message}") - imageProxy.close() - } - } - }) + imageAnalysis.setAnalyzer(ContextCompat.getMainExecutor(this), { imageProxy -> + val rotationDegrees = imageProxy.imageInfo.rotationDegrees + val image = imageProxy.image + if (image != null) { + val processImage = InputImage.fromMediaImage(image, rotationDegrees) + var element: DrawExercise + val result: Task = poseDetector + .process(processImage) + .addOnSuccessListener { pose -> + if(isActive && !isFinished){ + if (binding.parentLayout.childCount > 2) binding.parentLayout.removeViews( + 2, + binding.parentLayout.childCount - 2 + ) + element = DrawExercise( + ExerciseType.DumbellCurl_Type, + context = this, + pose, + isActive = true + ) + isFinished = element.giveIsFinished() + binding.parentLayout.addView(element) + } + imageProxy.close() + }.addOnFailureListener { + Log.v("MainActivity", "Error - ${it.message}") + imageProxy.close() + } + } + }) cameraProvider.bindToLifecycle(this as LifecycleOwner, cameraSelector, imageAnalysis, preview) } } \ No newline at end of file diff --git a/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/Draw.kt b/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/Draw.kt index ebc54a0..e2a557d 100644 --- a/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/Draw.kt +++ b/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/Draw.kt @@ -1,13 +1,20 @@ package com.example.mltestapplication.new +import android.app.Activity import android.content.Context +import android.content.res.Configuration import android.graphics.* +import android.os.Build +import android.util.DisplayMetrics import android.util.Log import android.view.View import android.widget.Toast import com.google.mlkit.vision.pose.Pose import java.lang.Exception + + + class Draw(context: Context) : View(context) { //context: Context?, var pose : Pose, val jointsMap : Map, val linesList : List>, var errorText : MutableList //, val redJointsMap : Map,val sets: Int, val reps : Int @@ -18,18 +25,21 @@ class Draw(context: Context) : View(context) { lateinit var linePaint: Paint lateinit var repBoxPaint: Paint - var xOffset = -275f - var yOffset = 250f - var scalex = 1.5f - var scaley = 1.5f - + var xOffset = -525 + var yOffset = 100 + var scalex = 2f + var scaley = 2f + var widthPixels = resources.displayMetrics.widthPixels init { init() } val widthRatio = 1280f/480f val heightRatio = 720f/360f - +// getWindowManager().getDefaultDisplay().getMetrics(metrics); +// width1 = metrics.widthPixels; +// height1 = metrics.heightPixels; +// private fun init(){ try { pointCorrectPaint = Paint() @@ -43,7 +53,7 @@ class Draw(context: Context) : View(context) { textPaint = Paint() textPaint.color = Color.BLACK textPaint.style = Paint.Style.FILL - textPaint.textSize =50f + textPaint.textSize =100f linePaint = Paint() linePaint.color = Color.GREEN @@ -71,13 +81,13 @@ class Draw(context: Context) : View(context) { "widthPixels = ${resources.displayMetrics.widthPixels}\n") try{ for (joint in jointsMap){ - val x = joint.value.x * scalex + xOffset - val y = joint.value.y * scaley + yOffset + val x = adjustX(joint.value.x) + val y = adjustY(joint.value.y) canvas.drawCircle(x,y,10f, pointCorrectPaint) } for (joint in redJointsMap){ - val x = joint.value.x * scalex + xOffset - val y = joint.value.y * scaley + yOffset + val x = adjustX(joint.value.x) + val y = adjustY(joint.value.y) canvas.drawCircle(x,y,10f, pointIncorrectPaint) } }catch (e: Exception){ @@ -89,10 +99,10 @@ class Draw(context: Context) : View(context) { try{ for (line in linesList) { - var point1x = (line.first?.x ?: 5000f) * scalex + xOffset - var point1y = (line.first?.y ?: 5000f) * scaley + yOffset - var point2x = (line.second?.x ?: 5000f) * scalex + xOffset - var point2y = (line.second?.y ?: 5000f) * scaley + yOffset + var point1x = adjustX(line.first?.x ?: 5000f) + var point1y = adjustY(line.first?.y ?: 5000f) + var point2x = adjustX(line.second?.x ?: 5000f) + var point2y = adjustY(line.second?.y ?: 5000f) if(point1x === 5000f || point1y === 5000f || point2x === 5000f || point2y === 5000f){ return } @@ -100,13 +110,21 @@ class Draw(context: Context) : View(context) { } }catch (e: Exception){ Log.e("drawLines", "Problem in drawLines of Draw class , msg: $e") - Toast.makeText(context,"Error in drawLines, check logs", Toast.LENGTH_LONG) + Toast.makeText(context,"Error in drawLines,S check logs", Toast.LENGTH_LONG) } } public fun drawRepsAndSets(canvas: Canvas, reps : Int, sets: Int){ - canvas.drawRect(700f,0f,1000f,200f,repBoxPaint ) - canvas.drawText("Reps: $reps",710f,20f,textPaint) - canvas.drawText("Sets: $sets",710f,80f,textPaint) + canvas.drawRect(700f,100f,1200f,500f,repBoxPaint ) + canvas.drawText("Reps: $reps",710f,200f,textPaint) + canvas.drawText("Sets: $sets",710f,300f,textPaint) + } + + private fun adjustX(x: Float):Float{ + return (x * scalex) + xOffset; + } + private fun adjustY(y: Float):Float{ + return (y * scaley) + yOffset; } } + diff --git a/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/DrawExercise.kt b/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/DrawExercise.kt index 714359b..7064068 100644 --- a/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/DrawExercise.kt +++ b/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/DrawExercise.kt @@ -3,13 +3,17 @@ package com.example.mltestapplication.new import android.content.Context import android.graphics.Canvas import android.view.View +import android.view.ViewParent import com.google.mlkit.vision.pose.Pose //This is a custom view -class DrawExercise(exerciseType: ExerciseType,context: Context,pose: Pose): View(context) { - var exercise = Factory().getExercise(ExerciseType.DumbellCurl_Type,context, pose) +class DrawExercise(exerciseType: ExerciseType,context: Context,pose: Pose,isActive : Boolean): View(context) { + var exercise = Factory().getExercise(ExerciseType.DumbellCurl_Type,context, pose,isActive) override fun onDraw(canvas: Canvas?) { var newCanvas = exercise?.drawing(canvas) super.onDraw(newCanvas) } + fun giveIsFinished(): Boolean{ + return exercise?.giveIsFinished() ?: false + } } \ No newline at end of file diff --git a/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/DumbellCurls.kt b/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/DumbellCurls.kt index 0223567..40be23f 100644 --- a/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/DumbellCurls.kt +++ b/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/DumbellCurls.kt @@ -9,7 +9,7 @@ import com.google.mlkit.vision.pose.Pose import com.google.mlkit.vision.pose.PoseLandmark import kotlin.math.log -class DumbellCurls(context: Context, val poseInput: Pose) : View(context),Exercise { +class DumbellCurls(context: Context, val poseInput: Pose, var isActive :Boolean) : View(context),Exercise { var isLeft : Boolean = false override val type: ExerciseType = ExerciseType.DumbellCurl_Type @@ -21,15 +21,22 @@ class DumbellCurls(context: Context, val poseInput: Pose) : View(context),Exerci private var shoulderAngle : Float = 0f var errorText: MutableList = MutableList(0){""} + //temporary parameters, these will be entered by the user in future +// Maximum number of reps in a set + var maxReps = 12 +// Maximum number of sets in total + var maxSets = 3 + + + var exception = false init { - init() +// if(isActive) + init() } fun init(){ try { - isLeft = poseInput.getPoseLandmark(PoseLandmark.LEFT_WRIST).position.x <= poseInput.getPoseLandmark( - PoseLandmark.LEFT_ELBOW - ).position.x + checkElbowPosition() if (isLeft){ jointsToDisplay = mutableMapOf( "SHOULDER" to poseInput.getPoseLandmark(PoseLandmark.LEFT_SHOULDER).position, @@ -74,10 +81,19 @@ class DumbellCurls(context: Context, val poseInput: Pose) : View(context),Exerci //two booleans used to set if last sequence was down-up , up-down var downUp = false var upDown = false + // + var isFinished = false } override fun drawing(canvas: Canvas?):Canvas { - if(!exception){ + if(setCount>=maxSets){ + isActive=false + isFinished = true + } + +// if(!exception && isActive){ + if(!exception){ + init() giveReps() giveRepPosition() giveErrorText() @@ -90,6 +106,10 @@ class DumbellCurls(context: Context, val poseInput: Pose) : View(context),Exerci } return canvas!! } + + override fun giveIsFinished(): Boolean { + return isFinished + } //return reps count fun giveReps() : Int{ @@ -104,6 +124,10 @@ class DumbellCurls(context: Context, val poseInput: Pose) : View(context),Exerci upDown = true } lastRepPosition = currentRepPosition + if(repCount>maxReps){ + ++setCount + repCount=0 + } return repCount } @@ -119,25 +143,38 @@ class DumbellCurls(context: Context, val poseInput: Pose) : View(context),Exerci } private fun giveErrorText(): MutableList{ + errorText.clear() if (elbowAngle<30){ errorText.add("Contracting Biceps Too much") - }else if(elbowAngle>160){ + } + if(elbowAngle>160){ errorText.add("Extending elbow too much") } - else{ - errorText.clear() + if(shoulderAngle>10){ + errorText.add("Elbow to forward") } return errorText } private fun giveRedJoints(): MutableMap{ + redJointsMap.clear() if (elbowAngle<30){ redJointsMap.put("ELBOW",jointsToDisplay["ELBOW"]!!) - }else if(elbowAngle>135){ + } + if(elbowAngle>135){ redJointsMap.put("ELBOW",jointsToDisplay["ELBOW"]!!) } - else{ - redJointsMap.clear() + if(shoulderAngle>10){ + redJointsMap.put("SHOULDER",jointsToDisplay["SHOULDER"]!!) + redJointsMap.put("ELBOW",jointsToDisplay["ELBOW"]!!) + } return redJointsMap } + + private fun checkElbowPosition(){ + isLeft = poseInput.getPoseLandmark(PoseLandmark.LEFT_WRIST).position.x <= poseInput.getPoseLandmark( + PoseLandmark.LEFT_ELBOW + ).position.x + } + } \ No newline at end of file diff --git a/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/Exercise.kt b/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/Exercise.kt index 5a71b47..8e49c45 100644 --- a/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/Exercise.kt +++ b/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/Exercise.kt @@ -5,9 +5,10 @@ import android.graphics.PointF interface Exercise { abstract fun drawing(canvas: Canvas?): Canvas? - + abstract fun giveIsFinished(): Boolean val type : ExerciseType var jointsToDisplay : MutableMap var linesToDisplay : MutableList> + // fun draw() } \ No newline at end of file diff --git a/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/Factory.kt b/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/Factory.kt index 4dc18c1..c4ef59e 100644 --- a/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/Factory.kt +++ b/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/Factory.kt @@ -4,9 +4,9 @@ import android.content.Context import com.google.mlkit.vision.pose.Pose class Factory { - fun getExercise(exType:ExerciseType,context: Context,pose: Pose) : Exercise? { + fun getExercise(exType:ExerciseType,context: Context,pose: Pose,isActive: Boolean) : Exercise? { return when(exType){ - ExerciseType.DumbellCurl_Type -> DumbellCurls(context,pose) + ExerciseType.DumbellCurl_Type -> DumbellCurls(context,pose,isActive) else -> null } } diff --git a/MLTestApplication/app/src/main/res/layout/activity_main.xml b/MLTestApplication/app/src/main/res/layout/activity_main.xml index af29c5c..88ba4a8 100644 --- a/MLTestApplication/app/src/main/res/layout/activity_main.xml +++ b/MLTestApplication/app/src/main/res/layout/activity_main.xml @@ -10,12 +10,36 @@ android:id="@+id/parentLayout" > +