Feature/audio

.gitignore

@@ -5,9 +5,11 @@
 *.swp
 .DS_Store
 .atom/
+.build/
 .buildlog/
 .history
 .svn/
+.swiftpm/
 migrate_working_dir/
 
 # IntelliJ related

analysis_options.yaml

@@ -11,7 +11,7 @@ include: package:flutter_lints/flutter.yaml
 
 analyzer:
   plugins:
-    - custom_lint
+  #- custom_lint
 
 linter:
   # The lint rules applied to this project can be customized in the
@@ -25,8 +25,8 @@ linter:
   # `// ignore_for_file: name_of_lint` syntax on the line or in the file
   # producing the lint.
   rules:
-    # avoid_print: false  # Uncomment to disable the `avoid_print` rule
-    # prefer_single_quotes: true  # Uncomment to enable the `prefer_single_quotes` rule
+  # avoid_print: false  # Uncomment to disable the `avoid_print` rule
+  # prefer_single_quotes: true  # Uncomment to enable the `prefer_single_quotes` rule
 
 # Additional information about this file can be found at
 # https://dart.dev/guides/language/analysis-options

devtools_options.yaml

@@ -0,0 +1,3 @@
+description: This file stores settings for Dart & Flutter DevTools.
+documentation: https://docs.flutter.dev/tools/devtools/extensions#configure-extension-enablement-states
+extensions:

lib/src/audio/notifiers/voice_notifier.dart

@@ -0,0 +1,59 @@
+/*
+ * SPDX-FileCopyrightText: 2024 Deutsche Telekom AG
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import 'dart:typed_data';
+
+import 'package:arc_view/src/audio/services/voice_recorder.dart';
+import 'package:arc_view/src/conversation/notifiers/conversations_notifier.dart';
+import 'package:audioplayers/audioplayers.dart';
+import 'package:riverpod_annotation/riverpod_annotation.dart';
+
+part 'voice_notifier.g.dart';
+
+@riverpod
+class VoiceNotifier extends _$VoiceNotifier {
+  VoiceRecorder? _voiceRecorder;
+
+  @override
+  VoiceStatus build() {
+    ref.onDispose(() {
+      stopRecording();
+    });
+    return VoiceStatus.idle;
+  }
+
+  playAudio(Uint8List audio) {
+    var player = AudioPlayer();
+    player.setSourceBytes(audio).then((a) {
+      player.resume();
+    });
+    player.onPlayerStateChanged.listen((event) {
+      if (event == PlayerState.completed) {
+        player.dispose();
+      }
+    });
+  }
+
+  startRecording() async {
+    _voiceRecorder = VoiceRecorder();
+    final data = await _voiceRecorder?.startRecordingStream();
+    if (data == null) return;
+    state = VoiceStatus.recording;
+    await ref.read(conversationsNotifierProvider.notifier).addUserVoice(data);
+    stopRecording();
+  }
+
+  stopRecording() async {
+    await _voiceRecorder?.stopRecording();
+    state = VoiceStatus.idle;
+  }
+}
+
+enum VoiceStatus {
+  idle,
+  recording,
+  error,
+}

lib/src/audio/services/voice_recorder.dart

@@ -0,0 +1,45 @@
+/*
+ * SPDX-FileCopyrightText: 2024 Deutsche Telekom AG
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import 'dart:typed_data';
+
+import 'package:http/http.dart' as http;
+import 'package:record/record.dart';
+
+class VoiceRecorder {
+  final _recordConfig = const RecordConfig(
+    numChannels: 1,
+    sampleRate: 24000,
+    encoder: AudioEncoder.pcm16bits,
+  );
+
+  final _record = AudioRecorder();
+
+  Future<Uint8List?> startRecording() async {
+    if (await _record.hasPermission()) {
+      await _record.start(_recordConfig, path: 'audio.pcm');
+      return Future.delayed(const Duration(seconds: 3), () async {
+        final path = await _record.stop();
+        final response = await http.get(Uri.parse(path!));
+        Uint8List data = response.bodyBytes;
+        _record.dispose();
+        return data;
+      });
+    }
+    return null;
+  }
+
+  Future<Stream<Uint8List>?> startRecordingStream() async {
+    if (await _record.hasPermission()) {
+      final stream = await _record.startStream(_recordConfig);
+      return stream;
+    }
+    return null;
+  }
+
+  Future<void> stopRecording() async {
+    await _record.stop();
+  }
+}

lib/src/chat/address_bar.dart

@@ -5,6 +5,7 @@
  */
 
 import 'package:arc_view/src/client/notifiers/agent_client_notifier.dart';
+import 'package:arc_view/src/client/notifiers/agent_stream_client_notifier.dart';
 import 'package:flutter/material.dart';
 import 'package:flutter_riverpod/flutter_riverpod.dart';
 import 'package:smiles/smiles.dart';
@@ -38,6 +39,9 @@ class _AddressBarState extends State<AddressBar> {
               ),
               onChanged: (value) {
                 ref.read(agentClientNotifierProvider.notifier).setUrl(value);
+                ref
+                    .read(agentStreamClientNotifierProvider.notifier)
+                    .setUrl(value);
               },
             ).expand(),
           ],

lib/src/chat/buttons/record_button.dart

@@ -0,0 +1,35 @@
+/*
+ * SPDX-FileCopyrightText: 2024 Deutsche Telekom AG
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import 'package:arc_view/src/audio/notifiers/voice_notifier.dart';
+import 'package:arc_view/src/core/secondary_button.dart';
+import 'package:flutter/material.dart';
+import 'package:flutter_riverpod/flutter_riverpod.dart';
+
+//
+// Starts the recording of the voice message.
+//
+class RecordButton extends ConsumerWidget {
+  const RecordButton({super.key});
+
+  @override
+  Widget build(BuildContext context, WidgetRef ref) {
+    return SecondaryButton(
+      description: 'Record voice',
+      icon: VoiceStatus.recording == ref.watch(voiceNotifierProvider)
+          ? Icons.mic_off
+          : Icons.mic,
+      onPressed: () async {
+        final status = ref.read(voiceNotifierProvider);
+        if (VoiceStatus.idle == status) {
+          ref.read(voiceNotifierProvider.notifier).startRecording();
+        } else {
+          ref.read(voiceNotifierProvider.notifier).stopRecording();
+        }
+      },
+    );
+  }
+}

lib/src/chat/chat_panel.dart

@@ -4,6 +4,8 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+import 'package:arc_view/src/audio/notifiers/voice_notifier.dart';
+import 'package:arc_view/src/chat/buttons/record_button.dart';
 import 'package:arc_view/src/chat/buttons/send_message_button.dart';
 import 'package:arc_view/src/chat/chat_field.dart';
 import 'package:arc_view/src/chat/chat_list.dart';
@@ -49,26 +51,33 @@ class _ChatPanelState extends State<ChatPanel> {
           children: [
             const ChatToolBar(),
             const ChatList().expand(),
-            Card(
-              elevation: 6,
-              margin: const EdgeInsets.fromLTRB(0, 16, 0, 16),
-              child: Row(
-                children: [
-                  _newConversationButton(ref, theme),
-                  _previousPromptButton(ref, theme),
-                  _chatField(ref),
-                  _sendButton(ref),
-                ],
-              ).padding(),
-            ),
+            [
+              Card(
+                elevation: 6,
+                margin: const EdgeInsets.fromLTRB(0, 16, 0, 16),
+                child: Row(
+                  children: [
+                    _newConversationButton(ref, theme),
+                    _previousPromptButton(ref, theme),
+                    _chatField(ref),
+                    SendMessageButton(onPressed: () => _send(ref)),
+                  ],
+                ).padding(),
+              ).size(height: 100).expand(),
+              HGap.small(),
+              Card(
+                elevation: 6,
+                margin: const EdgeInsets.fromLTRB(0, 16, 0, 16),
+                child: RecordButton().padding(),
+              ).size(height: 100),
+            ].row(),
           ],
         );
       },
     );
   }
 
-  _chatField(WidgetRef ref) =>
-      Expanded(
+  _chatField(WidgetRef ref) => Expanded(
         child: Consumer(builder: (ctx, ref, child) {
           _textController.text = ref.watch(currentPromptNotifierProvider);
           return ChatField(
@@ -78,18 +87,15 @@ class _ChatPanelState extends State<ChatPanel> {
         }),
       );
 
-  _previousPromptButton(WidgetRef ref, ThemeData theme) =>
-      IconButton(
-          icon: Icon(
-            Icons.line_weight_sharp,
-            color: theme.colorScheme.onSurface,
-          ),
-          onPressed: () {
-            //ref.read(currentPromptControllerProvider.notifier).rotate();
-            showPromptList(context);
-          }).tip('Show previous prompts');
-
-  _sendButton(WidgetRef ref) => SendMessageButton(onPressed: () => _send(ref));
+  _previousPromptButton(WidgetRef ref, ThemeData theme) => IconButton(
+      icon: Icon(
+        Icons.line_weight_sharp,
+        color: theme.colorScheme.onSurface,
+      ),
+      onPressed: () {
+        //ref.read(currentPromptControllerProvider.notifier).rotate();
+        showPromptList(context);
+      }).tip('Show previous prompts');
 
   _send(WidgetRef ref) {
     if (_textController.text.isEmpty) return;
@@ -101,8 +107,7 @@ class _ChatPanelState extends State<ChatPanel> {
     _textController.text = '';
   }
 
-  _newConversationButton(WidgetRef ref, ThemeData theme) =>
-      IconButton(
+  _newConversationButton(WidgetRef ref, ThemeData theme) => IconButton(
         icon: Icon(
           Icons.add,
           color: theme.colorScheme.onSurface,

lib/src/chat/message/chat_message_card.dart

@@ -23,7 +23,10 @@ class ChatMessageCard extends StatelessWidget {
       margin: const EdgeInsets.all(8),
       child: Stack(
         children: [
-          chatMessage.content.txt.pad(16, 16, 32, 16),
+          if (chatMessage.isBinary())
+            'Loading transcript...'.txt.pad(16, 16, 32, 16),
+          if (!chatMessage.isBinary())
+            chatMessage.content.txt.pad(16, 16, 32, 16),
           Positioned(
             bottom: 0,
             right: 0,

lib/src/client/message.dart

@@ -0,0 +1,35 @@
+/*
+ * SPDX-FileCopyrightText: 2024 Deutsche Telekom AG
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import 'package:freezed_annotation/freezed_annotation.dart';
+
+part 'message.freezed.dart';
+part 'message.g.dart';
+
+///
+/// Message Type from the Arc API.
+///
+@freezed
+class AgentResult with _$AgentResult {
+  factory AgentResult({
+    required List<Message> messages,
+    required double responseTime,
+  }) = _AgentResult;
+
+  factory AgentResult.fromJson(Map<String, Object?> json) =>
+      _$AgentResultFromJson(json);
+}
+
+@freezed
+class Message with _$Message {
+  factory Message({
+    required String role,
+    required String content,
+  }) = _Message;
+
+  factory Message.fromJson(Map<String, Object?> json) =>
+      _$MessageFromJson(json);
+}