pull to docker

.github/workflows/DiscordVirtualAssistant.yml

@@ -33,3 +33,4 @@ jobs:
         env:
           PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }}
           CLIENT_EMAIL: ${{ secrets.CLIENT_EMAIL }}
+          DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }}

.gitignore

@@ -137,4 +137,6 @@ node_modules/
 response.mp3
 config.json
 recordings/
-GoogleServiceAccount.json
+GoogleServiceAccount.json
+token.txt
+scripts/figures/*.png

index.js

@@ -6,46 +6,105 @@
  @Description: Main file for program.
 
  @Changelog:
- 3/19/2022 IS:Convert main loop into async function and add sleep function.
- 2/25/2022 IS:Added import statement for config file and import for environmental variables as needed.
- 2/19/2022 IS:Added import statements and basic operation.
+ 3/19/2022 IS: Add audio conversion with ffmpeg.
+ 3/19/2022 IS: Convert main loop into async function and add sleep function.
+ 2/25/2022 IS: Added import statement for config file and import for environmental variables as needed.
+ 2/19/2022 IS: Added import statements and basic operation.
  */
+// Discord handler
 const DiscordHandler = require('./src/discord');
 const Parser = require('./src/parser');
 const S2T = require('./src/s2t');
+const T2S = require('./src/t2s');
+const { execSync } = require('child_process');
 
-const discordclient = new DiscordHandler();
-const parser = new Parser(discordclient);
+const discord_client = new DiscordHandler();
+const parser = new Parser(discord_client);
 const speech2text = new S2T();
+const text2speech = new T2S();
 
 /**
  * Helper function, sleeps for x ms.
- * @param ms miliseconds to sleep
+ * @param ms {int} Milliseconds to sleep
  */
 function sleep(ms) {
 	return new Promise((resolve) => {
 		setTimeout(resolve, ms);
 	});
 }
 
+/**
+ * Use ffmpeg to convert audio from signed-16 bit, little endian, 48000hz, 2 channel audio to WAV
+ * @param audio {string} File name of audio file
+ * @returns {string} New audio filename
+ */
+function convert_audio(audio) {
+	const new_filename = audio.slice(0, -4) + '.wav';
+	console.log('audio conversion has started');
+	// use ffmpeg to convert signed-16 bit, little endian, 48000hz, 2 channel audio to WAV
+	const command = 'ffmpeg -f s16le -ar 48k -ac 2 -i ' + audio + ' ' + new_filename;
+	console.log(command);
+	execSync(command);
+	console.log('audio conversion has ended');
+	return new_filename;
+}
 
-discordclient.login().then(async () => {
-	let cont = true;
+discord_client.login().then(async () => {
+	const cont = true;
 	while (cont) {
 		// if the discord client has audio ready to process
-		if (discordclient.audio_ready) {
+		if (discord_client.audio_status) {
+			let audio = await discord_client.audio_clip;
+			await sleep(1);
+			audio = convert_audio(audio);
+
 			// transcribe the audio using the speech to text module
-			const transcript = await speech2text.transcribe(discordclient.audio_clip);
+			const transcript = await speech2text.transcribe(audio);
+
+			console.log('Transcript:' + transcript);
+
 			// Parse the transcript and preform actions
 			const status = parser.parse(transcript);
 
-			if (!status) {
-				cont = false;
+			let test = 'none';
+			switch (status) {
+			case 'Played file':
+				test = 'play';
+				break;
+			case 'Stated time':
+				test = 'time';
+				break;
+			case 'Stated weather':
+				test = 'weather';
+				break;
+			case 'Banned user':
+				test = 'ban';
+				break;
+			case 'DMed user':
+				test = 'dm';
+				break;
+			case 'no command found':
+				test = 'none';
+				break;
+			case 'echoed command':
+				test = 'echo';
+				break;
 			}
+
+			await discord_client.pool.getConnection()
+				.then (conn => {
+					conn.query('USE dva');
+					conn.query('REPLACE INTO command_usage VALUES (?)', [test]);
+					return conn.release();
+				});
+
+			// await text2speech.convert(status);
+
+			await discord_client.play('response.mp3');
 		}
 		await sleep(100);
 	}
 },
 ).finally(() => {
-	discordclient.logout().then(() => console.log('Bot shutdown successfully'));
+	discord_client.logout().then(() => console.log('Bot shutdown successfully'));
 });

scripts/figures/README.md

@@ -0,0 +1,17 @@
+# Figures
+
+## Instructions
+
+### Installation
+Figure generation is handled by python if you do not have a python interpreter or the pip package manager they can be downloaded [here](https://www.python.org/downloads/)
+1. install required python packages
+```shell
+python -m pip install -r requirements.txt
+```
+2. Figure generation requires database access, if you haven't setup database environmental variables please refer to the main README for instructions
+### Usage
+To run figure generation:
+```shell
+python generate_figures.py
+```
+This will save and display a graph which represents the usage of the record function over time.

scripts/figures/generate_figures.py

@@ -0,0 +1,110 @@
+import mariadb
+import sys
+import pandas as pd
+import matplotlib.pyplot as plt
+import matplotlib.dates as mdates
+import os
+
+"""
+ @Project: Discord Virtual Assistant
+ @Title: generate_figures.py
+ @Authors: Ian Sodersjerna, Jacob Austin, Jonathan Tucker, Nick Miceli
+ @Created: 2/19/2022
+ @Description: Creates figures to track usage of DVA
+
+ @Changelog:
+ 3/29/2022 IS: Basic database connection, data processing, and figure generation
+"""
+
+try:
+    conn = mariadb.connect(
+        user=os.environ.get('DVA_DATABASE_USER'),
+        password=os.environ.get('DVA_DATABASE_PASSWORD'),
+        host=os.environ.get('DVA_DATABASE_HOST'),
+        database="dva"
+
+    )
+except mariadb.Error as e:
+    print(f"Error connecting to MariaDB Platform: {e}")
+    sys.exit(1)
+
+# Get Cursor
+cur = conn.cursor()
+
+cur.execute("SELECT time FROM record_usage")
+
+dates = []
+for t in cur:
+    dates.append(t[0].date())
+
+usage_per_day = []
+while len(dates) > 0:
+    count = 0
+    d0 = dates[0]
+
+    # count occurrences of d0
+    for d in dates:
+        if d == d0:
+            count += 1
+
+    # remove all occurrences of d0
+    dates = [i for i in dates if i != d0]
+
+    usage_per_day.append((d0, count))
+
+usage_per_day.sort()
+
+day, usage = zip(*usage_per_day)
+
+plt.bar(day, usage)
+plt.title("record command usage")
+plt.xlabel('Days')
+plt.xticks(rotation='vertical')
+
+plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%m/%d/%Y'))
+plt.gca().xaxis.set_major_locator(mdates.DayLocator())
+
+plt.ylabel('Usage')
+
+plt.autoscale()
+plt.subplots_adjust(bottom=0.25)
+
+plt.savefig("record_usage_figure.png")
+plt.show()
+
+cur.execute("SELECT command FROM command_usage")
+
+commands = []
+for command in cur:
+    commands.append(command[0])
+commands.sort()
+
+commands_usage = []
+while len(commands) > 0:
+    count = 0
+    c0 = commands[0]
+
+    # count occurrences of d0
+    for d in commands:
+        if d == c0:
+            count += 1
+
+    # remove all occurrences of d0
+    commands = [i for i in commands if i != c0]
+
+    commands_usage.append((c0, count))
+
+command, usage = zip(*commands_usage)
+
+plt.bar(command, usage)
+plt.title("voice command usage")
+plt.xlabel('command')
+
+
+plt.ylabel('Usage')
+
+plt.autoscale()
+# plt.subplots_adjust(bottom=0.25)
+
+plt.savefig("command_usage_figure.png")
+plt.show()

scripts/figures/requirements.txt

@@ -0,0 +1,3 @@
+matplotlib
+pandas
+mariadb