diff --git a/MLTestApplication/app/build.gradle b/MLTestApplication/app/build.gradle index ab8ff8c..646de8d 100644 --- a/MLTestApplication/app/build.gradle +++ b/MLTestApplication/app/build.gradle @@ -32,15 +32,15 @@ android { jvmTarget = '1.8' } - aaptOptions { - noCompress "tflite" - } buildFeatures{ viewBinding = true } dataBinding{ enabled = true } + androidResources { + noCompress 'tflite' + } } dependencies { diff --git a/MLTestApplication/app/src/main/java/com/example/mltestapplication/MainActivity.kt b/MLTestApplication/app/src/main/java/com/example/mltestapplication/MainActivity.kt index 3dfd934..3ebdccf 100644 --- a/MLTestApplication/app/src/main/java/com/example/mltestapplication/MainActivity.kt +++ b/MLTestApplication/app/src/main/java/com/example/mltestapplication/MainActivity.kt @@ -1,13 +1,20 @@ package com.example.mltestapplication -import androidx.appcompat.app.AppCompatActivity -import android.os.Bundle import android.annotation.SuppressLint +import android.os.Bundle +import android.os.CountDownTimer import android.util.Log import android.util.Size -import androidx.core.content.ContextCompat +import android.view.Gravity +import android.view.View +import android.view.ViewGroup +import android.widget.Button +import android.widget.LinearLayout +import android.widget.TextView +import androidx.appcompat.app.AppCompatActivity import androidx.camera.core.* import androidx.camera.lifecycle.ProcessCameraProvider +import androidx.core.content.ContextCompat import androidx.databinding.DataBindingUtil import androidx.lifecycle.LifecycleOwner import com.example.mltestapplication.databinding.ActivityMainBinding @@ -20,26 +27,35 @@ import com.google.mlkit.vision.pose.Pose import com.google.mlkit.vision.pose.PoseDetection import com.google.mlkit.vision.pose.PoseDetector import com.google.mlkit.vision.pose.defaults.PoseDetectorOptions +import kotlinx.android.synthetic.main.activity_main.* + class MainActivity : AppCompatActivity(){ private lateinit var binding: ActivityMainBinding private lateinit var poseDetector : PoseDetector private lateinit var cameraProviderFuture: ListenableFuture + private lateinit var countdown_timer: CountDownTimer + private lateinit var start_button: Button + private var isActive : Boolean = false +// private lateinit var textView : TextView + private var isFinished : Boolean = false + override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) binding = DataBindingUtil.setContentView(this,R.layout.activity_main) - //getSupportActionBar()?.hide() + supportActionBar?.hide() cameraProviderFuture = ProcessCameraProvider.getInstance(this) cameraProviderFuture.addListener({ - Log.d("CALLED", "Called inside cameraProviderFuture.addListener") val cameraProvider = cameraProviderFuture.get() bindPreview(cameraProvider = cameraProvider) }, ContextCompat.getMainExecutor(this)) + + // val localModel = LocalModel.Builder() // .setAbsoluteFilePath("pose_detection.tflite") // .build() @@ -49,16 +65,48 @@ class MainActivity : AppCompatActivity(){ .build() - poseDetector = PoseDetection.getClient(poseDetectorOptions) + start_button = findViewById(R.id.id_start_button) as Button + start_button.setOnClickListener{ + startTimer(5) + } + + id_stop_button.setOnClickListener{ + isActive = false + isFinished = true + if (binding.parentLayout.childCount > 2) binding.parentLayout.removeViews( + 2, + binding.parentLayout.childCount - 2 + ) + } + countdown.visibility = View.INVISIBLE + id_stop_button.visibility = View.INVISIBLE + + } + private fun startTimer(time_in_seconds: Long) { + binding.parentLayout.removeView(start_button) + countdown.visibility = View.VISIBLE + countdown_timer = object : CountDownTimer(time_in_seconds*1000, 1000) { + override fun onFinish() { + isActive = true + binding.parentLayout.removeView(countdown) + id_stop_button.visibility = View.VISIBLE + } + + override fun onTick(timeLeft: Long) { + countdown.text = (timeLeft/1000).toString() + } + } + countdown_timer.start() + + } @SuppressLint("UnsafeOptInUsageError") private fun bindPreview(cameraProvider: ProcessCameraProvider){ - Log.d("CALLED", "Called Just inside Bind preview") val preview = Preview.Builder().build() - +// when true, analyses, displays and updates sets and reps of exercise //defining the camera, the prerequisites before opening the camera val cameraSelector = CameraSelector.Builder() @@ -69,35 +117,42 @@ class MainActivity : AppCompatActivity(){ val imageAnalysis = ImageAnalysis.Builder() - .setTargetResolution(Size(1280,720)) + .setTargetResolution(Size(resources.displayMetrics.heightPixels / 2,resources.displayMetrics.widthPixels / 2 )) .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) .build() - imageAnalysis.setAnalyzer(ContextCompat.getMainExecutor(this),{imageProxy -> - Log.d("CALLED", "Called inside imageAnalysis.setAnalyzer") - val rotationDegrees = imageProxy.imageInfo.rotationDegrees - val image = imageProxy.image - if(image != null){ - val processImage = InputImage.fromMediaImage(image, rotationDegrees) - var element : DrawExercise - val result : Task = poseDetector - .process(processImage) - .addOnSuccessListener { pose -> -// val allPoseLandmarks = pose.allPoseLandmarks - Log.d("CALLED", "Called inside poseDetector.onSuccessListener") - if (binding.parentLayout.childCount > 1) binding.parentLayout.removeViews( 1, - binding.parentLayout.childCount - 1) - element =DrawExercise(ExerciseType.DumbellCurl_Type,context=this, pose) - binding.parentLayout.addView(element) - imageProxy.close() - }.addOnFailureListener { - Log.v("MainActivity", "Error - ${it.message}") - imageProxy.close() - } - } - }) + imageAnalysis.setAnalyzer(ContextCompat.getMainExecutor(this), { imageProxy -> + val rotationDegrees = imageProxy.imageInfo.rotationDegrees + val image = imageProxy.image + if (image != null) { + val processImage = InputImage.fromMediaImage(image, rotationDegrees) + var element: DrawExercise + val result: Task = poseDetector + .process(processImage) + .addOnSuccessListener { pose -> + if(isActive && !isFinished){ + if (binding.parentLayout.childCount > 2) binding.parentLayout.removeViews( + 2, + binding.parentLayout.childCount - 2 + ) + element = DrawExercise( + ExerciseType.DumbellCurl_Type, + context = this, + pose, + isActive = true + ) + isFinished = element.giveIsFinished() + binding.parentLayout.addView(element) + } + imageProxy.close() + }.addOnFailureListener { + Log.v("MainActivity", "Error - ${it.message}") + imageProxy.close() + } + } + }) cameraProvider.bindToLifecycle(this as LifecycleOwner, cameraSelector, imageAnalysis, preview) } } \ No newline at end of file diff --git a/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/Draw.kt b/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/Draw.kt index ebc54a0..e2a557d 100644 --- a/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/Draw.kt +++ b/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/Draw.kt @@ -1,13 +1,20 @@ package com.example.mltestapplication.new +import android.app.Activity import android.content.Context +import android.content.res.Configuration import android.graphics.* +import android.os.Build +import android.util.DisplayMetrics import android.util.Log import android.view.View import android.widget.Toast import com.google.mlkit.vision.pose.Pose import java.lang.Exception + + + class Draw(context: Context) : View(context) { //context: Context?, var pose : Pose, val jointsMap : Map, val linesList : List>, var errorText : MutableList //, val redJointsMap : Map,val sets: Int, val reps : Int @@ -18,18 +25,21 @@ class Draw(context: Context) : View(context) { lateinit var linePaint: Paint lateinit var repBoxPaint: Paint - var xOffset = -275f - var yOffset = 250f - var scalex = 1.5f - var scaley = 1.5f - + var xOffset = -525 + var yOffset = 100 + var scalex = 2f + var scaley = 2f + var widthPixels = resources.displayMetrics.widthPixels init { init() } val widthRatio = 1280f/480f val heightRatio = 720f/360f - +// getWindowManager().getDefaultDisplay().getMetrics(metrics); +// width1 = metrics.widthPixels; +// height1 = metrics.heightPixels; +// private fun init(){ try { pointCorrectPaint = Paint() @@ -43,7 +53,7 @@ class Draw(context: Context) : View(context) { textPaint = Paint() textPaint.color = Color.BLACK textPaint.style = Paint.Style.FILL - textPaint.textSize =50f + textPaint.textSize =100f linePaint = Paint() linePaint.color = Color.GREEN @@ -71,13 +81,13 @@ class Draw(context: Context) : View(context) { "widthPixels = ${resources.displayMetrics.widthPixels}\n") try{ for (joint in jointsMap){ - val x = joint.value.x * scalex + xOffset - val y = joint.value.y * scaley + yOffset + val x = adjustX(joint.value.x) + val y = adjustY(joint.value.y) canvas.drawCircle(x,y,10f, pointCorrectPaint) } for (joint in redJointsMap){ - val x = joint.value.x * scalex + xOffset - val y = joint.value.y * scaley + yOffset + val x = adjustX(joint.value.x) + val y = adjustY(joint.value.y) canvas.drawCircle(x,y,10f, pointIncorrectPaint) } }catch (e: Exception){ @@ -89,10 +99,10 @@ class Draw(context: Context) : View(context) { try{ for (line in linesList) { - var point1x = (line.first?.x ?: 5000f) * scalex + xOffset - var point1y = (line.first?.y ?: 5000f) * scaley + yOffset - var point2x = (line.second?.x ?: 5000f) * scalex + xOffset - var point2y = (line.second?.y ?: 5000f) * scaley + yOffset + var point1x = adjustX(line.first?.x ?: 5000f) + var point1y = adjustY(line.first?.y ?: 5000f) + var point2x = adjustX(line.second?.x ?: 5000f) + var point2y = adjustY(line.second?.y ?: 5000f) if(point1x === 5000f || point1y === 5000f || point2x === 5000f || point2y === 5000f){ return } @@ -100,13 +110,21 @@ class Draw(context: Context) : View(context) { } }catch (e: Exception){ Log.e("drawLines", "Problem in drawLines of Draw class , msg: $e") - Toast.makeText(context,"Error in drawLines, check logs", Toast.LENGTH_LONG) + Toast.makeText(context,"Error in drawLines,S check logs", Toast.LENGTH_LONG) } } public fun drawRepsAndSets(canvas: Canvas, reps : Int, sets: Int){ - canvas.drawRect(700f,0f,1000f,200f,repBoxPaint ) - canvas.drawText("Reps: $reps",710f,20f,textPaint) - canvas.drawText("Sets: $sets",710f,80f,textPaint) + canvas.drawRect(700f,100f,1200f,500f,repBoxPaint ) + canvas.drawText("Reps: $reps",710f,200f,textPaint) + canvas.drawText("Sets: $sets",710f,300f,textPaint) + } + + private fun adjustX(x: Float):Float{ + return (x * scalex) + xOffset; + } + private fun adjustY(y: Float):Float{ + return (y * scaley) + yOffset; } } + diff --git a/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/DrawExercise.kt b/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/DrawExercise.kt index 714359b..7064068 100644 --- a/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/DrawExercise.kt +++ b/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/DrawExercise.kt @@ -3,13 +3,17 @@ package com.example.mltestapplication.new import android.content.Context import android.graphics.Canvas import android.view.View +import android.view.ViewParent import com.google.mlkit.vision.pose.Pose //This is a custom view -class DrawExercise(exerciseType: ExerciseType,context: Context,pose: Pose): View(context) { - var exercise = Factory().getExercise(ExerciseType.DumbellCurl_Type,context, pose) +class DrawExercise(exerciseType: ExerciseType,context: Context,pose: Pose,isActive : Boolean): View(context) { + var exercise = Factory().getExercise(ExerciseType.DumbellCurl_Type,context, pose,isActive) override fun onDraw(canvas: Canvas?) { var newCanvas = exercise?.drawing(canvas) super.onDraw(newCanvas) } + fun giveIsFinished(): Boolean{ + return exercise?.giveIsFinished() ?: false + } } \ No newline at end of file diff --git a/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/DumbellCurls.kt b/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/DumbellCurls.kt index 0223567..40be23f 100644 --- a/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/DumbellCurls.kt +++ b/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/DumbellCurls.kt @@ -9,7 +9,7 @@ import com.google.mlkit.vision.pose.Pose import com.google.mlkit.vision.pose.PoseLandmark import kotlin.math.log -class DumbellCurls(context: Context, val poseInput: Pose) : View(context),Exercise { +class DumbellCurls(context: Context, val poseInput: Pose, var isActive :Boolean) : View(context),Exercise { var isLeft : Boolean = false override val type: ExerciseType = ExerciseType.DumbellCurl_Type @@ -21,15 +21,22 @@ class DumbellCurls(context: Context, val poseInput: Pose) : View(context),Exerci private var shoulderAngle : Float = 0f var errorText: MutableList = MutableList(0){""} + //temporary parameters, these will be entered by the user in future +// Maximum number of reps in a set + var maxReps = 12 +// Maximum number of sets in total + var maxSets = 3 + + + var exception = false init { - init() +// if(isActive) + init() } fun init(){ try { - isLeft = poseInput.getPoseLandmark(PoseLandmark.LEFT_WRIST).position.x <= poseInput.getPoseLandmark( - PoseLandmark.LEFT_ELBOW - ).position.x + checkElbowPosition() if (isLeft){ jointsToDisplay = mutableMapOf( "SHOULDER" to poseInput.getPoseLandmark(PoseLandmark.LEFT_SHOULDER).position, @@ -74,10 +81,19 @@ class DumbellCurls(context: Context, val poseInput: Pose) : View(context),Exerci //two booleans used to set if last sequence was down-up , up-down var downUp = false var upDown = false + // + var isFinished = false } override fun drawing(canvas: Canvas?):Canvas { - if(!exception){ + if(setCount>=maxSets){ + isActive=false + isFinished = true + } + +// if(!exception && isActive){ + if(!exception){ + init() giveReps() giveRepPosition() giveErrorText() @@ -90,6 +106,10 @@ class DumbellCurls(context: Context, val poseInput: Pose) : View(context),Exerci } return canvas!! } + + override fun giveIsFinished(): Boolean { + return isFinished + } //return reps count fun giveReps() : Int{ @@ -104,6 +124,10 @@ class DumbellCurls(context: Context, val poseInput: Pose) : View(context),Exerci upDown = true } lastRepPosition = currentRepPosition + if(repCount>maxReps){ + ++setCount + repCount=0 + } return repCount } @@ -119,25 +143,38 @@ class DumbellCurls(context: Context, val poseInput: Pose) : View(context),Exerci } private fun giveErrorText(): MutableList{ + errorText.clear() if (elbowAngle<30){ errorText.add("Contracting Biceps Too much") - }else if(elbowAngle>160){ + } + if(elbowAngle>160){ errorText.add("Extending elbow too much") } - else{ - errorText.clear() + if(shoulderAngle>10){ + errorText.add("Elbow to forward") } return errorText } private fun giveRedJoints(): MutableMap{ + redJointsMap.clear() if (elbowAngle<30){ redJointsMap.put("ELBOW",jointsToDisplay["ELBOW"]!!) - }else if(elbowAngle>135){ + } + if(elbowAngle>135){ redJointsMap.put("ELBOW",jointsToDisplay["ELBOW"]!!) } - else{ - redJointsMap.clear() + if(shoulderAngle>10){ + redJointsMap.put("SHOULDER",jointsToDisplay["SHOULDER"]!!) + redJointsMap.put("ELBOW",jointsToDisplay["ELBOW"]!!) + } return redJointsMap } + + private fun checkElbowPosition(){ + isLeft = poseInput.getPoseLandmark(PoseLandmark.LEFT_WRIST).position.x <= poseInput.getPoseLandmark( + PoseLandmark.LEFT_ELBOW + ).position.x + } + } \ No newline at end of file diff --git a/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/Exercise.kt b/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/Exercise.kt index 5a71b47..8e49c45 100644 --- a/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/Exercise.kt +++ b/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/Exercise.kt @@ -5,9 +5,10 @@ import android.graphics.PointF interface Exercise { abstract fun drawing(canvas: Canvas?): Canvas? - + abstract fun giveIsFinished(): Boolean val type : ExerciseType var jointsToDisplay : MutableMap var linesToDisplay : MutableList> + // fun draw() } \ No newline at end of file diff --git a/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/Factory.kt b/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/Factory.kt index 4dc18c1..c4ef59e 100644 --- a/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/Factory.kt +++ b/MLTestApplication/app/src/main/java/com/example/mltestapplication/new/Factory.kt @@ -4,9 +4,9 @@ import android.content.Context import com.google.mlkit.vision.pose.Pose class Factory { - fun getExercise(exType:ExerciseType,context: Context,pose: Pose) : Exercise? { + fun getExercise(exType:ExerciseType,context: Context,pose: Pose,isActive: Boolean) : Exercise? { return when(exType){ - ExerciseType.DumbellCurl_Type -> DumbellCurls(context,pose) + ExerciseType.DumbellCurl_Type -> DumbellCurls(context,pose,isActive) else -> null } } diff --git a/MLTestApplication/app/src/main/res/layout/activity_main.xml b/MLTestApplication/app/src/main/res/layout/activity_main.xml index af29c5c..88ba4a8 100644 --- a/MLTestApplication/app/src/main/res/layout/activity_main.xml +++ b/MLTestApplication/app/src/main/res/layout/activity_main.xml @@ -10,12 +10,36 @@ android:id="@+id/parentLayout" > +