From 1c707b18b68a5f71934acc5f5ac3dfd26a978faf Mon Sep 17 00:00:00 2001 From: charlieforward9 <62311337+charlieforward9@users.noreply.github.com> Date: Sat, 2 Sep 2023 16:28:18 -0400 Subject: [PATCH 1/3] fix #93 : [Inference] initial classification --- lib/bloc/inference/inference_bloc.dart | 5 +++-- lib/views/components/video_controls.dart | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/bloc/inference/inference_bloc.dart b/lib/bloc/inference/inference_bloc.dart index e8acc714..983c5247 100644 --- a/lib/bloc/inference/inference_bloc.dart +++ b/lib/bloc/inference/inference_bloc.dart @@ -21,7 +21,7 @@ class InferenceBloc extends Bloc { on(_onInput, transformer: concurrent()); on(_onOutput, transformer: concurrent()); on(_onEnd); - classification = false; + classification = true; storageBacklog = 0; //TODO dynamically load models through constructor @@ -88,8 +88,9 @@ class InferenceBloc extends Bloc { _inferenceService.store( event.input, classification, event.ratio, event.output!); } - if (!isClosed && !emit.isDone) + if (!isClosed && !emit.isDone) { emit(InferenceRunning(event.output, event.ratio)); + } } catch (e) { log("Error: $e"); } diff --git a/lib/views/components/video_controls.dart b/lib/views/components/video_controls.dart index cc0fba78..d00d6c3a 100644 --- a/lib/views/components/video_controls.dart +++ b/lib/views/components/video_controls.dart @@ -17,7 +17,7 @@ class VideoControls extends StatefulWidget { } class _VideoControlsState extends State { - bool classification = false; + bool classification = true; @override Widget build(BuildContext context) { final inf = BlocProvider.of(context), From fdedd59d942682ec0f732caa967bea7bd20037ed Mon Sep 17 00:00:00 2001 From: charlieforward9 <62311337+charlieforward9@users.noreply.github.com> Date: Sun, 3 Sep 2023 19:12:57 -0400 Subject: [PATCH 2/3] fix #77 [Validation] Improved Controls --- lib/bloc/validation/validation_bloc.dart | 64 ++++++++-- lib/bloc/validation/validation_event.dart | 17 ++- lib/bloc/validation/validation_state.dart | 6 +- lib/main.dart | 22 ++-- lib/presenter/validation_presenter.dart | 10 +- lib/service/storage_service.dart | 14 +++ lib/views/app_nav.dart | 8 +- lib/views/home_view.dart | 49 ++++++-- lib/views/validation_view.dart | 138 ++++++++++++++-------- 9 files changed, 231 insertions(+), 97 deletions(-) diff --git a/lib/bloc/validation/validation_bloc.dart b/lib/bloc/validation/validation_bloc.dart index 0a5fa788..eaea4d9a 100644 --- a/lib/bloc/validation/validation_bloc.dart +++ b/lib/bloc/validation/validation_bloc.dart @@ -3,7 +3,6 @@ import 'dart:developer'; import 'dart:ui'; import 'package:amplify_flutter/amplify_flutter.dart' as amplify; -import 'package:amplify_storage_s3/amplify_storage_s3.dart'; import 'package:flutter_bloc/flutter_bloc.dart'; import 'package:equatable/equatable.dart'; import 'package:visualpt/service/storage_service.dart'; @@ -14,9 +13,14 @@ part 'validation_state.dart'; class ValidationBloc extends Bloc { final storage = StorageService(); + // Internal list to keep track of processed IDs + final List _processedKeys = []; + // Internal list to keep track of procesed labels + final List _processedInputs = []; ValidationBloc() : super(const ValidationLoad([])) { on(_onFetch); + on(_onPrev); on(_onNext); on(_onInput); on(_onEnd); @@ -28,6 +32,39 @@ class ValidationBloc extends Bloc { add(ValidationNext(keys: storedKeys)); } + FutureOr _onPrev( + ValidationPrev event, Emitter emit) async { + //Put the ID back into the list of unprocessed IDs + String id = _processedKeys.removeLast(); + int classification = _processedInputs.removeLast(); + event.keys.insert(0, event.currentKey); + event.keys.insert(0, id); + if (classification == 1) { + storage.move( + sourceKey: 'valid/$id.csv', + destinationKey: 'raw/$id.csv', + sourceAccessLevel: amplify.StorageAccessLevel.guest, + destinationAccessLevel: amplify.StorageAccessLevel.private); + storage.move( + sourceKey: 'valid/$id.csv', + destinationKey: 'raw/$id.csv', + sourceAccessLevel: amplify.StorageAccessLevel.guest, + destinationAccessLevel: amplify.StorageAccessLevel.private); + } else { + storage.move( + sourceKey: 'invalid/$id.csv', + destinationKey: 'raw/$id.csv', + sourceAccessLevel: amplify.StorageAccessLevel.guest, + destinationAccessLevel: amplify.StorageAccessLevel.private); + storage.move( + sourceKey: 'invalid/$id.csv', + destinationKey: 'raw/$id.csv', + sourceAccessLevel: amplify.StorageAccessLevel.guest, + destinationAccessLevel: amplify.StorageAccessLevel.private); + } + add(ValidationNext(keys: event.keys)); + } + FutureOr _onNext( ValidationNext event, Emitter emit) async { String imageID, dataID; @@ -51,7 +88,7 @@ class ValidationBloc extends Bloc { } } else { imageID = temp; - searchKey = temp.split('.').first; + searchKey = temp.split('.').first.split('/').last; final dataIndex = event.keys.indexWhere((element) => element.contains(searchKey)); if (dataIndex != -1) { @@ -60,6 +97,7 @@ class ValidationBloc extends Bloc { throw Exception('No data file found'); } } + //Get the image and data from the storage final imageResult = await storage .getUrl(imageID, amplify.StorageAccessLevel.private) .result; @@ -81,7 +119,7 @@ class ValidationBloc extends Bloc { emit(ValidationDisplay( event.keys, key: searchKey, - originalDataBytes: dataResult.bytes, + dataBytes: dataResult.bytes, imageUrl: imageResult.url.toString(), imageHeight: imageHeight, imageWidth: imageWidth, @@ -100,13 +138,25 @@ class ValidationBloc extends Bloc { FutureOr _onInput( ValidationInput event, Emitter emit) { + _processedKeys.add(event.key); + _processedInputs.add(event.input ? 1 : 0); //Store the validated data if (event.input) { - storage.upload(S3DataPayload.bytes(event.originalDataBytes), - 'valid/${event.key}.csv', amplify.StorageAccessLevel.guest); + //Store the ID of the image and data file in a list in case we need to go back + storage.move( + sourceKey: 'raw/${event.key}.csv', + destinationKey: 'valid/${event.key}.csv'); + storage.move( + sourceKey: 'raw/${event.key}.png', + destinationKey: 'valid/${event.key}.png'); + } else { + storage.move( + sourceKey: 'raw/${event.key}.csv', + destinationKey: 'invalid/${event.key}.csv'); + storage.move( + sourceKey: 'raw/${event.key}.png', + destinationKey: 'invalid/${event.key}.png'); } - storage.remove('raw/${event.key}.csv', amplify.StorageAccessLevel.private); - storage.remove('raw/${event.key}.png', amplify.StorageAccessLevel.private); //Start the next validation add(ValidationNext(keys: event.keys)); } diff --git a/lib/bloc/validation/validation_event.dart b/lib/bloc/validation/validation_event.dart index 50f8b2d6..1b930c71 100644 --- a/lib/bloc/validation/validation_event.dart +++ b/lib/bloc/validation/validation_event.dart @@ -14,27 +14,36 @@ class ValidationFetch extends ValidationEvent { List get props => []; } -class ValidationNext extends ValidationEvent { +class ValidationPrev extends ValidationEvent { + const ValidationPrev({required this.currentKey, required this.keys}); + final String currentKey; final List keys; + + @override + List get props => [keys]; +} + +class ValidationNext extends ValidationEvent { const ValidationNext({required this.keys}); + final List keys; @override List get props => [keys]; } class ValidationInput extends ValidationEvent { - final List originalDataBytes; + final List dataBytes; final String key; final List keys; final bool input; const ValidationInput( - {required this.originalDataBytes, + {required this.dataBytes, required this.key, required this.keys, required this.input}); @override - List get props => [originalDataBytes, input]; + List get props => [dataBytes, input]; } class ValidationEnd extends ValidationEvent { diff --git a/lib/bloc/validation/validation_state.dart b/lib/bloc/validation/validation_state.dart index 37f73e29..8cf6955d 100644 --- a/lib/bloc/validation/validation_state.dart +++ b/lib/bloc/validation/validation_state.dart @@ -17,7 +17,7 @@ class ValidationLoad extends ValidationState { class ValidationDisplay extends ValidationState { final String key; - final List originalDataBytes; + final List dataBytes; final String imageUrl; final int imageHeight; final int imageWidth; @@ -26,7 +26,7 @@ class ValidationDisplay extends ValidationState { final List offsets; const ValidationDisplay(super.keys, {required this.key, - required this.originalDataBytes, + required this.dataBytes, required this.imageUrl, required this.imageHeight, required this.imageWidth, @@ -36,7 +36,7 @@ class ValidationDisplay extends ValidationState { @override List get props => - [imageUrl, imageHeight, imageWidth, classification, offsets]; + [dataBytes, imageUrl, imageHeight, imageWidth, classification, offsets]; } class ValidationDispose extends ValidationState { diff --git a/lib/main.dart b/lib/main.dart index 2bfad2b6..46e6b8f5 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -50,13 +50,11 @@ class VisualPT extends StatelessWidget { fontFamily: "Unbounded", color: CupertinoColors.black))), routes: { "/": (context) => const LandingView(), - "/home": (context) => const HomeView(), - "/video": (context) => VideoView( + "/home": (context) => HomeView( (BlocProvider.of(context).state as AuthSuccess) .user - .email != - 'gaitertech@gmail.com', //If the email is gaitertech@gmail.com, they are a guest, DO NOT let them store to database - (_) {}), + .email == + 'gaitertech@gmail.com'), "/validate": (context) => const ValidationView(), "/settings": (context) => const SettingsView(), }, @@ -67,11 +65,15 @@ class VisualPT extends StatelessWidget { builder: (context) => PatientView(assessmentType: assessmentType)); } - // if (route.name == "/video") { - // final args = route.arguments; - // return CupertinoPageRoute( - // builder: (context) => VideoView(onComplete: args.onComplete)); - // } + if (route.name == "/video") { + final args = route.arguments as Map; + final storeData = args["storageAccess"] as bool; + final onComplete = + args["onComplete"] as dynamic Function(Segment); + + return CupertinoPageRoute( + builder: (context) => VideoView(storeData, onComplete)); + } return null; }, ), diff --git a/lib/presenter/validation_presenter.dart b/lib/presenter/validation_presenter.dart index 122c1356..37ff6e6e 100644 --- a/lib/presenter/validation_presenter.dart +++ b/lib/presenter/validation_presenter.dart @@ -1,24 +1,28 @@ import 'package:flutter/cupertino.dart'; import 'package:flutter_bloc/flutter_bloc.dart'; import 'package:visualpt/bloc/validation/validation_bloc.dart'; -import 'package:visualpt/views/app_nav.dart'; /// Provided by ValidationBloc class ValidationPresenter extends StatelessWidget { final Widget Function(BuildContext context, ValidationDisplay state) onDisplay; final Widget Function(BuildContext context, ValidationLoad state) onLoad; + final Widget Function(BuildContext context, ValidationDispose state) + onDispose; final Widget Function(BuildContext context, ValidationError state) onError; //TODO: add error handling for... /* +* " * The following HttpException was thrown resolving an image codec: * Connection closed before full header was received +* " **/ const ValidationPresenter({ super.key, required this.onDisplay, required this.onLoad, + required this.onDispose, required this.onError, }); @@ -38,9 +42,7 @@ class ValidationPresenter extends StatelessWidget { } else if (state is ValidationDisplay) { return onDisplay(context, state); } else if (state is ValidationDispose) { - AppNav(context).back(); - //TODO: Fix this - return Container(); + return onDispose(context, state); } else { throw Exception("Undefined Validation State"); } diff --git a/lib/service/storage_service.dart b/lib/service/storage_service.dart index d890f574..c0296c26 100644 --- a/lib/service/storage_service.dart +++ b/lib/service/storage_service.dart @@ -52,6 +52,20 @@ class StorageService { } } + void move( + {required String sourceKey, + required String destinationKey, + StorageAccessLevel sourceAccessLevel = StorageAccessLevel.private, + StorageAccessLevel destinationAccessLevel = StorageAccessLevel.private}) { + Amplify.Storage.move( + source: StorageItemWithAccessLevel( + storageItem: StorageItem(key: sourceKey), + accessLevel: sourceAccessLevel), + destination: StorageItemWithAccessLevel( + storageItem: StorageItem(key: destinationKey), + accessLevel: destinationAccessLevel)); + } + void remove(String key, StorageAccessLevel accessLevel) { Amplify.Storage.remove( key: key, options: StorageRemoveOptions(accessLevel: accessLevel)); diff --git a/lib/views/app_nav.dart b/lib/views/app_nav.dart index 4130ab3e..2b4d86c4 100644 --- a/lib/views/app_nav.dart +++ b/lib/views/app_nav.dart @@ -14,13 +14,13 @@ class AppNav { }); } - toPath(String path) { + void toPath({required String path, Object? args}) { Future(() async { - Navigator.of(context).pushNamed(path); + Navigator.of(context).pushNamed(path, arguments: args); }); } - toView(Widget view) { + void toView(Widget view) { Future(() async { Navigator.of(context).push( CupertinoPageRoute( @@ -30,7 +30,7 @@ class AppNav { }); } - reset() { + void reset() { Future(() async { Navigator.pushNamedAndRemoveUntil(context, '/', (route) => false); }); diff --git a/lib/views/home_view.dart b/lib/views/home_view.dart index dc3b40aa..4c0179d9 100644 --- a/lib/views/home_view.dart +++ b/lib/views/home_view.dart @@ -1,11 +1,14 @@ import 'package:flutter/cupertino.dart'; +import 'package:flutter_bloc/flutter_bloc.dart'; import 'package:visualpt/models/ModelProvider.dart'; import 'package:visualpt/views/_views.dart'; import 'package:visualpt/views/app_nav.dart'; +import '../bloc/_bloc.dart'; import 'components/_components.dart'; class HomeView extends StatelessWidget { - const HomeView({super.key}); + const HomeView(this.isDemo, {super.key}); + final bool isDemo; @override Widget build(BuildContext context) { @@ -14,7 +17,6 @@ class HomeView extends StatelessWidget { return CupertinoPageScaffold( child: BaseView( child: Column( - mainAxisSize: MainAxisSize.min, children: [ //TODO: Abstract the styling into a AssessmentCardList widget Flexible( @@ -45,18 +47,39 @@ class HomeView extends StatelessWidget { ), ), ), - CupertinoButton( - onPressed: () => {nav.toPath("/validate")}, - color: Styles.actionPrimary, - child: const Text("Validate Data", style: Styles.actionText)), + + isDemo + ? Container() + : Padding( + padding: const EdgeInsets.all(8.0), + child: CupertinoButton( + onPressed: () => nav.toPath(path: "/video", args: { + "storageAccess": + (BlocProvider.of(context).state + as AuthSuccess) + .user + .email != + 'gaitertech@gmail.com', + "onComplete": (Segment s) => {nav.back()} + }), + color: Styles.actionPrimary, + child: const Text( + "Collect Data ", + style: Styles.actionText, + )), + ), + + isDemo + ? Container() + : Padding( + padding: const EdgeInsets.all(8.0), + child: CupertinoButton( + onPressed: () => {nav.toPath(path: "/validate")}, + color: Styles.actionPrimary, + child: const Text("Validate Data", + style: Styles.actionText)), + ), const Spacer(), - // CupertinoButton( - // onPressed: () => nav.toPath("/video"), - // color: Styles.comingSoonColor, - // child: const Text( - // "Test Video", - // style: Styles.error, - // )), ], ), ), diff --git a/lib/views/validation_view.dart b/lib/views/validation_view.dart index 0e7a8297..636103c5 100644 --- a/lib/views/validation_view.dart +++ b/lib/views/validation_view.dart @@ -3,6 +3,7 @@ import 'package:flutter_bloc/flutter_bloc.dart'; import 'package:visualpt/bloc/validation/validation_bloc.dart'; import 'package:visualpt/presenter/_presenter.dart'; import 'package:visualpt/views/_views.dart'; +import 'package:visualpt/views/app_nav.dart'; import 'package:visualpt/views/components/_components.dart'; import 'camera_painters/pose_painter.dart'; @@ -38,8 +39,10 @@ class ValidationView extends StatelessWidget { children: [ const ViewBackground(), ValidationPresenter( + //onCollect: (context, state) => VideoView(true, (_) => {}), onDisplay: (context, state) => onDisplay(context, state), onLoad: (context, state) => const AppLoading(), + onDispose: (context, state) => onDispose(context, state), //TODO: Create AppError widget for all blocks onError: (context, state) => onError(context, state), ) @@ -49,60 +52,91 @@ class ValidationView extends StatelessWidget { } Widget onDisplay(BuildContext context, ValidationDisplay state) { - return ConstrainedBox( - constraints: BoxConstraints.tightFor( - height: state.imageHeight.toDouble(), - width: state.imageWidth.toDouble()), - child: Stack( - alignment: Alignment.center, - fit: StackFit.expand, - children: [ - Image.network( - state.imageUrl, - height: state.imageHeight.toDouble(), - width: state.imageWidth.toDouble(), - fit: BoxFit - .fitWidth, // This ensures that the image covers the entire space defined by height and width + return Column( + children: [ + ConstrainedBox( + constraints: BoxConstraints.tightFor( + height: state.imageHeight.toDouble() * state.ratio, + width: state.imageWidth.toDouble() * state.ratio), + child: Stack( + alignment: Alignment.center, + fit: StackFit.expand, + children: [ + Image.network( + state.imageUrl, + height: state.imageHeight.toDouble(), + width: state.imageWidth.toDouble(), + fit: BoxFit + .fitWidth, // This ensures that the image covers the entire space defined by height and width + ), + drawPose(state.offsets.map((o) => o * state.ratio).toList()), + Positioned( + right: 0, + top: 0, + child: Container( + color: CupertinoColors.black.withOpacity(0.5), + child: Text(state.classification == 1 ? "Static" : "Sway", + style: Styles.actionText)), + ), + ], + )), + ConstrainedBox( + constraints: BoxConstraints( + maxHeight: 100, maxWidth: MediaQuery.of(context).size.width), + child: Column(children: [ + Row( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + CupertinoButton( + color: CupertinoColors.systemBlue, + child: const Text("True"), + onPressed: () => BlocProvider.of(context) + .add(ValidationInput( + key: state.key, + dataBytes: state.dataBytes, + keys: state.keys, + input: true))), + const Spacer(), + CupertinoButton( + color: CupertinoColors.systemBlue, + child: const Text("False"), + onPressed: () => BlocProvider.of(context) + .add(ValidationInput( + key: state.key, + dataBytes: state.dataBytes, + keys: state.keys, + input: false))), + ], + ), + const Spacer(), + CupertinoButton( + onPressed: () => BlocProvider.of(context) + .add(ValidationPrev(currentKey: state.key, keys: state.keys)), + child: const Text( + "Previous Image", + style: Styles.error, + ), ), - drawPose(state.offsets), - Positioned( - bottom: 40, - child: ConstrainedBox( - constraints: BoxConstraints( - maxHeight: 100, - maxWidth: MediaQuery.of(context).size.width), - child: Column(children: [ - Text(state.classification == 1 ? "Static" : "Sway", - style: Styles.subactionText), - Row( - mainAxisAlignment: MainAxisAlignment.center, - children: [ - CupertinoButton( - child: const Text("True"), - onPressed: () => - BlocProvider.of(context).add( - ValidationInput( - key: state.key, - originalDataBytes: - state.originalDataBytes, - keys: state.keys, - input: true))), - CupertinoButton( - child: const Text("False"), - onPressed: () => - BlocProvider.of(context).add( - ValidationInput( - key: state.key, - originalDataBytes: - state.originalDataBytes, - keys: state.keys, - input: false))), - ], - ) - ]), - )) + ]), + ) + ], + ); + } + + Widget onDispose(BuildContext context, ValidationDispose state) { + return SizedBox( + height: MediaQuery.of(context).size.height, + width: MediaQuery.of(context).size.width, + child: Center( + child: Column( + children: [ + const Text("No more data to validate", style: Styles.subactionText), + CupertinoButton( + onPressed: () => AppNav(context).back(), + child: const Text("Go to home"), + ) ], - )); + ))); } onError(BuildContext _, ValidationError state) { From 7845976684454b1a30242095a0ceb5f7d3cce530 Mon Sep 17 00:00:00 2001 From: charlieforward9 <62311337+charlieforward9@users.noreply.github.com> Date: Sun, 3 Sep 2023 19:13:03 -0400 Subject: [PATCH 3/3] minor: styles --- lib/views/video_view.dart | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/lib/views/video_view.dart b/lib/views/video_view.dart index d9825e59..40e9282e 100644 --- a/lib/views/video_view.dart +++ b/lib/views/video_view.dart @@ -14,12 +14,17 @@ class VideoView extends StatelessWidget { final Function(Segment) onComplete; @override Widget build(BuildContext context) { - return VideoPresenter( - storageAccess, - onRecording: recordingVideo, - onStandby: standbyVideo, - onLoading: loadingVideo, - onError: errorVideo, + return Stack( + children: [ + const ViewBackground(), + VideoPresenter( + storageAccess, + onRecording: recordingVideo, + onStandby: standbyVideo, + onLoading: loadingVideo, + onError: errorVideo, + ), + ], ); }