From 4c2776131d630020bcca89381966e039e77eec5d Mon Sep 17 00:00:00 2001 From: jurei733 <67505990+jurei733@users.noreply.github.com> Date: Sun, 23 Mar 2025 18:09:42 +0100 Subject: [PATCH 001/143] Add keycloak auth guard --- apps/frontend/src/app/app.routes.ts | 26 +++++++++--- apps/frontend/src/app/auth/auth.guard.ts | 31 ++++++++++++++ .../src/app/auth/keycloak-initializer.ts | 40 ------------------- .../src/app/auth/service/auth.service.spec.ts | 4 +- 4 files changed, 53 insertions(+), 48 deletions(-) create mode 100644 apps/frontend/src/app/auth/auth.guard.ts delete mode 100755 apps/frontend/src/app/auth/keycloak-initializer.ts diff --git a/apps/frontend/src/app/app.routes.ts b/apps/frontend/src/app/app.routes.ts index a1cddbd03..4b3d9cc29 100755 --- a/apps/frontend/src/app/app.routes.ts +++ b/apps/frontend/src/app/app.routes.ts @@ -13,17 +13,30 @@ import { TestGroupsComponent } from './ws-admin/components/test-groups/test-grou import { WsUsersComponent } from './ws-admin/components/ws-users/ws-users.component'; import { CodingManagementComponent } from './coding/coding-managment/coding-management.component'; import { CodingManagementManualComponent } from './coding/coding-management-manual/coding-management-manual.component'; +import { canActivateAuthRole } from './auth/auth.guard'; export const routes: Routes = [ { path: '', redirectTo: 'home', pathMatch: 'full' }, - { path: 'home', component: HomeComponent }, - { path: 'replay/:testPerson/:unitId/:page', component: ReplayComponent }, - { path: 'replay/:testPerson/:unitId', component: ReplayComponent }, - { path: 'replay/:testPerson', component: ReplayComponent }, - { path: 'replay', component: ReplayComponent }, - { path: 'coding-manual', component: CodingManagementManualComponent }, + { + path: 'home', + component: HomeComponent + }, + { + path: 'replay/:testPerson/:unitId/:page', + canActivate: [canActivateAuthRole], + component: ReplayComponent + }, + { + path: 'replay/:testPerson/:unitId', + canActivate: [canActivateAuthRole], + component: ReplayComponent + }, + { path: 'replay/:testPerson', canActivate: [canActivateAuthRole], component: ReplayComponent }, + { path: 'replay', canActivate: [canActivateAuthRole], component: ReplayComponent }, + { path: 'coding-manual', canActivate: [canActivateAuthRole], component: CodingManagementManualComponent }, { path: 'admin', + canActivate: [canActivateAuthRole], component: AdminComponent, children: [ { path: '', redirectTo: 'users', pathMatch: 'full' }, @@ -34,6 +47,7 @@ export const routes: Routes = [ { path: '**', component: UsersComponent }] }, { path: 'workspace-admin/:ws', + canActivate: [canActivateAuthRole], component: WsAdminComponent, children: [ { path: '', redirectTo: 'select-unit-play', pathMatch: 'full' }, diff --git a/apps/frontend/src/app/auth/auth.guard.ts b/apps/frontend/src/app/auth/auth.guard.ts new file mode 100644 index 000000000..9009bdda1 --- /dev/null +++ b/apps/frontend/src/app/auth/auth.guard.ts @@ -0,0 +1,31 @@ +import { + ActivatedRouteSnapshot, RouterStateSnapshot, Router, CanActivateFn, UrlTree +} from '@angular/router'; +import { inject } from '@angular/core'; +import { createAuthGuard, AuthGuardData } from 'keycloak-angular'; + +const isAccessAllowed = async ( + route: ActivatedRouteSnapshot, + _: RouterStateSnapshot, + authData: AuthGuardData +): Promise => { + const { authenticated, grantedRoles } = authData; + + // @ts-ignore + const requiredRole = route.data.role; + if (!requiredRole) { + return false; + } + + const hasRequiredRole = (role: string): boolean => Object.values(grantedRoles.resourceRoles) + .some(roles => roles.includes(role)); + + if (authenticated && hasRequiredRole(requiredRole)) { + return true; + } + + const router = inject(Router); + return router.parseUrl('/forbidden'); +}; + +export const canActivateAuthRole = createAuthGuard(isAccessAllowed); diff --git a/apps/frontend/src/app/auth/keycloak-initializer.ts b/apps/frontend/src/app/auth/keycloak-initializer.ts deleted file mode 100755 index 0f295dda3..000000000 --- a/apps/frontend/src/app/auth/keycloak-initializer.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { KeycloakOptions, KeycloakService } from 'keycloak-angular'; -import { environment } from '../../environments/environment'; - -export function initializer(keycloak: KeycloakService): () => Promise { - if (environment.production) { - const options: KeycloakOptions = { - config: { - url: 'https://www.iqb-login.de', - realm: 'iqb', - clientId: 'coding-box' - }, - loadUserProfileAtStartUp: true, - initOptions: { - onLoad: 'check-sso', - // redirectUri: 'https://iqb-kodierbox.de', - // onLoad: 'login-required', - checkLoginIframe: false - } - }; - - return () => keycloak.init(options); - } - const options: KeycloakOptions = { - config: { - url: 'https://www.iqb-login.de', - realm: 'iqb', - clientId: 'coding-box' - }, - loadUserProfileAtStartUp: true, - initOptions: { - onLoad: 'check-sso', - // onLoad: 'login-required', - checkLoginIframe: false - } - // enableBearerInterceptor: true, - // bearerExcludedUrls: ['replay'] - }; - - return () => keycloak.init(options); -} diff --git a/apps/frontend/src/app/auth/service/auth.service.spec.ts b/apps/frontend/src/app/auth/service/auth.service.spec.ts index 0513e3fbd..e283c43e1 100755 --- a/apps/frontend/src/app/auth/service/auth.service.spec.ts +++ b/apps/frontend/src/app/auth/service/auth.service.spec.ts @@ -1,6 +1,6 @@ import { TestBed } from '@angular/core/testing'; import { TranslateModule } from '@ngx-translate/core'; -import { HttpClientModule } from '@angular/common/http'; +import { provideHttpClient } from '@angular/common/http'; import { AuthService } from './auth.service'; describe('AuthService', () => { @@ -10,7 +10,7 @@ describe('AuthService', () => { TestBed.configureTestingModule({ imports: [ TranslateModule.forRoot(), - HttpClientModule + provideHttpClient() ], providers: [ { From 22453c040a0998fc88dad4f2ba64725941ae3b92 Mon Sep 17 00:00:00 2001 From: jurei733 <67505990+jurei733@users.noreply.github.com> Date: Mon, 24 Mar 2025 13:37:44 +0100 Subject: [PATCH 002/143] Allow frontend commonJs dependencies --- apps/frontend/project.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apps/frontend/project.json b/apps/frontend/project.json index f71b9ce1b..5d62765fb 100755 --- a/apps/frontend/project.json +++ b/apps/frontend/project.json @@ -11,6 +11,11 @@ "{options.outputPath}" ], "options": { + "allowedCommonJsDependencies": [ + "xml2js", + "base64-js", + "js-sha256" + ], "outputPath": "dist/apps/frontend", "index": "apps/frontend/src/index.html", "main": "apps/frontend/src/main.ts", From d749d7a2c7911c7631806730b63df8b16545d4c9 Mon Sep 17 00:00:00 2001 From: jurei733 <67505990+jurei733@users.noreply.github.com> Date: Mon, 24 Mar 2025 13:41:32 +0100 Subject: [PATCH 003/143] Redo guard to canActivateAuth --- apps/frontend/src/app/app.routes.ts | 16 ++++++++-------- apps/frontend/src/app/auth/auth.guard.ts | 23 ++++------------------- 2 files changed, 12 insertions(+), 27 deletions(-) diff --git a/apps/frontend/src/app/app.routes.ts b/apps/frontend/src/app/app.routes.ts index 4b3d9cc29..e89ae565d 100755 --- a/apps/frontend/src/app/app.routes.ts +++ b/apps/frontend/src/app/app.routes.ts @@ -13,7 +13,7 @@ import { TestGroupsComponent } from './ws-admin/components/test-groups/test-grou import { WsUsersComponent } from './ws-admin/components/ws-users/ws-users.component'; import { CodingManagementComponent } from './coding/coding-managment/coding-management.component'; import { CodingManagementManualComponent } from './coding/coding-management-manual/coding-management-manual.component'; -import { canActivateAuthRole } from './auth/auth.guard'; +import { canActivateAuth } from './auth/auth.guard'; export const routes: Routes = [ { path: '', redirectTo: 'home', pathMatch: 'full' }, @@ -23,20 +23,20 @@ export const routes: Routes = [ }, { path: 'replay/:testPerson/:unitId/:page', - canActivate: [canActivateAuthRole], + canActivate: [canActivateAuth], component: ReplayComponent }, { path: 'replay/:testPerson/:unitId', - canActivate: [canActivateAuthRole], + canActivate: [canActivateAuth], component: ReplayComponent }, - { path: 'replay/:testPerson', canActivate: [canActivateAuthRole], component: ReplayComponent }, - { path: 'replay', canActivate: [canActivateAuthRole], component: ReplayComponent }, - { path: 'coding-manual', canActivate: [canActivateAuthRole], component: CodingManagementManualComponent }, + { path: 'replay/:testPerson', canActivate: [canActivateAuth], component: ReplayComponent }, + { path: 'replay', canActivate: [canActivateAuth], component: ReplayComponent }, + { path: 'coding-manual', canActivate: [canActivateAuth], component: CodingManagementManualComponent }, { path: 'admin', - canActivate: [canActivateAuthRole], + canActivate: [canActivateAuth], component: AdminComponent, children: [ { path: '', redirectTo: 'users', pathMatch: 'full' }, @@ -47,7 +47,7 @@ export const routes: Routes = [ { path: '**', component: UsersComponent }] }, { path: 'workspace-admin/:ws', - canActivate: [canActivateAuthRole], + canActivate: [canActivateAuth], component: WsAdminComponent, children: [ { path: '', redirectTo: 'select-unit-play', pathMatch: 'full' }, diff --git a/apps/frontend/src/app/auth/auth.guard.ts b/apps/frontend/src/app/auth/auth.guard.ts index 9009bdda1..859982cdc 100644 --- a/apps/frontend/src/app/auth/auth.guard.ts +++ b/apps/frontend/src/app/auth/auth.guard.ts @@ -1,7 +1,6 @@ import { - ActivatedRouteSnapshot, RouterStateSnapshot, Router, CanActivateFn, UrlTree + ActivatedRouteSnapshot, RouterStateSnapshot, CanActivateFn, UrlTree } from '@angular/router'; -import { inject } from '@angular/core'; import { createAuthGuard, AuthGuardData } from 'keycloak-angular'; const isAccessAllowed = async ( @@ -9,23 +8,9 @@ const isAccessAllowed = async ( _: RouterStateSnapshot, authData: AuthGuardData ): Promise => { - const { authenticated, grantedRoles } = authData; + const { authenticated } = authData; - // @ts-ignore - const requiredRole = route.data.role; - if (!requiredRole) { - return false; - } - - const hasRequiredRole = (role: string): boolean => Object.values(grantedRoles.resourceRoles) - .some(roles => roles.includes(role)); - - if (authenticated && hasRequiredRole(requiredRole)) { - return true; - } - - const router = inject(Router); - return router.parseUrl('/forbidden'); + return authenticated; }; -export const canActivateAuthRole = createAuthGuard(isAccessAllowed); +export const canActivateAuth = createAuthGuard(isAccessAllowed); From 003e4afee2bef245b35ec73416d36dfdc4672e99 Mon Sep 17 00:00:00 2001 From: jurei733 <67505990+jurei733@users.noreply.github.com> Date: Mon, 24 Mar 2025 13:42:51 +0100 Subject: [PATCH 004/143] Refactor app component --- apps/frontend/src/app/app.component.html | 79 +++++++++++-------- apps/frontend/src/app/app.component.scss | 17 ++++ apps/frontend/src/app/app.component.ts | 45 +++++++---- .../components/app-info/app-info.component.ts | 8 +- 4 files changed, 97 insertions(+), 52 deletions(-) diff --git a/apps/frontend/src/app/app.component.html b/apps/frontend/src/app/app.component.html index 0bc4b1e94..3ca19f021 100755 --- a/apps/frontend/src/app/app.component.html +++ b/apps/frontend/src/app/app.component.html @@ -1,44 +1,55 @@
@if (appService.dataLoading) { - + } + + +
diff --git a/apps/frontend/src/app/app.component.scss b/apps/frontend/src/app/app.component.scss index 080c838ac..6fa9a82c9 100755 --- a/apps/frontend/src/app/app.component.scss +++ b/apps/frontend/src/app/app.component.scss @@ -27,3 +27,20 @@ img{ .app-header{ padding:10px; } + +.app-title { + color: lightgrey ; + text-align: center; + margin: 0 auto; + + + &.margin-logged-in { + margin-left: 25px; + margin-right: 0; + } + + &.margin-logged-out { + margin-left: 0; + margin-right: 100px; + } +} diff --git a/apps/frontend/src/app/app.component.ts b/apps/frontend/src/app/app.component.ts index 8d6f0e3a5..98b6995be 100755 --- a/apps/frontend/src/app/app.component.ts +++ b/apps/frontend/src/app/app.component.ts @@ -8,6 +8,7 @@ import { TranslateModule } from '@ngx-translate/core'; import { MatTooltip } from '@angular/material/tooltip'; import { MatButton } from '@angular/material/button'; import { LocationStrategy } from '@angular/common'; +import { KeycloakProfile } from 'keycloak-js'; import { AppService } from './services/app.service'; import { AuthService } from './auth/service/auth.service'; import { CreateUserDto } from '../../../../api-dto/user/create-user-dto'; @@ -42,27 +43,43 @@ export class AppComponent implements OnInit { async ngOnInit(): Promise { if (this.authService.isLoggedIn()) { - this.loggedInKeycloak = true; - this.appService.isLoggedInKeycloak = true; - this.appService.loggedUser = this.authService.getLoggedUser(); - this.appService.userProfile = await this.authService.loadUserProfile(); + this.setAuthState(); + + const keycloakUserProfile = await this.authService.loadUserProfile(); const isAdmin = this.authService.getRoles().includes('admin'); - if (this.appService.userProfile.id && this.appService.userProfile.username) { - const keycloakUser: CreateUserDto = { - issuer: this.appService.loggedUser?.iss || '', - identity: this.appService.userProfile.id, - username: this.appService.userProfile.username, - lastName: this.appService.userProfile.lastName || '', - firstName: this.appService.userProfile.firstName || '', - email: this.appService.userProfile.email || '', - isAdmin: isAdmin - }; + + if (this.isValidUserProfile(keycloakUserProfile)) { + const keycloakUser = this.createKeycloakUser(keycloakUserProfile, isAdmin); this.appService.kcUser = keycloakUser; await this.keycloakLogin(keycloakUser); } } + window.addEventListener('message', event => { this.appService.processMessagePost(event); }, false); } + + private setAuthState(): void { + this.loggedInKeycloak = true; + this.appService.isLoggedInKeycloak = true; + this.appService.loggedUser = this.authService.getLoggedUser(); + } + + // eslint-disable-next-line class-methods-use-this + private isValidUserProfile(userProfile: KeycloakProfile): boolean { + return !!userProfile?.id && !!userProfile?.username; + } + + private createKeycloakUser(userProfile: KeycloakProfile, isAdmin: boolean): CreateUserDto { + return { + issuer: this.appService.loggedUser?.iss || '', + identity: userProfile.id, + username: userProfile.username || '', + lastName: userProfile.lastName || '', + firstName: userProfile.firstName || '', + email: userProfile.email || '', + isAdmin: isAdmin + }; + } } diff --git a/apps/frontend/src/app/components/app-info/app-info.component.ts b/apps/frontend/src/app/components/app-info/app-info.component.ts index b62689dc8..2beff24d3 100755 --- a/apps/frontend/src/app/components/app-info/app-info.component.ts +++ b/apps/frontend/src/app/components/app-info/app-info.component.ts @@ -5,10 +5,10 @@ import { RouterLink } from '@angular/router'; import { MatAnchor } from '@angular/material/button'; @Component({ - selector: 'coding-box-app-info', - templateUrl: './app-info.component.html', - styleUrls: ['./app-info.component.scss'], - imports: [MatAnchor, RouterLink, TranslateModule] + selector: 'coding-box-app-info', + templateUrl: './app-info.component.html', + styleUrls: ['./app-info.component.scss'], + imports: [MatAnchor, RouterLink, TranslateModule] }) export class AppInfoComponent { @Input() appTitle!: string; From 3dada3679b817c3f8807abb7688b920e0444444d Mon Sep 17 00:00:00 2001 From: jurei733 <67505990+jurei733@users.noreply.github.com> Date: Mon, 24 Mar 2025 14:09:10 +0100 Subject: [PATCH 005/143] Make components subscribe on authData in app service --- apps/frontend/src/app/app.component.html | 2 +- apps/frontend/src/app/app.component.ts | 9 +++- .../app/components/home/home.component.html | 37 ++++++++-------- .../src/app/components/home/home.component.ts | 42 +++++++++++++------ apps/frontend/src/app/services/app.service.ts | 25 +++++------ .../src/app/services/backend.service.ts | 2 +- .../components/users/users.component.ts | 11 +++-- .../user-workspaces-area.component.html | 4 +- .../user-workspaces-area.component.ts | 2 - .../components/ws-admin/ws-admin.component.ts | 6 ++- 10 files changed, 86 insertions(+), 54 deletions(-) diff --git a/apps/frontend/src/app/app.component.html b/apps/frontend/src/app/app.component.html index 3ca19f021..c39c2758f 100755 --- a/apps/frontend/src/app/app.component.html +++ b/apps/frontend/src/app/app.component.html @@ -29,7 +29,7 @@
- @if (appService.authData.isAdmin || authService.getRoles().includes('admin')) { + @if (authData.isAdmin || authService.getRoles().includes('admin')) {
- + - - ID: {{ item.id }} + + play_arrow + {{ booklet.id }} {{ unit?.alias }} @@ -79,19 +80,17 @@ -

Grouped Blocks

- -
-

Block {{ i + 1 }}

-
    -
  • - {{ log.key }} - {{ log.parameter }}
    - Timestamp: {{ formatTimestamp(log.ts) }} -
    • -
-
- - +
+ play_arrow + @for ( logBlock of this.logs; track logBlock;) { + @for ( log of logBlock; track log;) { +
+ {{ log.key }} - {{ log.parameter }} + {{ formatTimestamp(log.ts) }} +
+ } + }
+ } diff --git a/apps/frontend/src/app/ws-admin/components/test-results/test-results.component.ts b/apps/frontend/src/app/ws-admin/components/test-results/test-results.component.ts index d390b4938..724cf8966 100755 --- a/apps/frontend/src/app/ws-admin/components/test-results/test-results.component.ts +++ b/apps/frontend/src/app/ws-admin/components/test-results/test-results.component.ts @@ -20,10 +20,13 @@ import { } from '@angular/material/expansion'; import { MatList, MatListItem } from '@angular/material/list'; import { MatTooltip } from '@angular/material/tooltip'; +import { MatInput } from '@angular/material/input'; +import { CommonModule, DatePipe } from '@angular/common'; +import { MatIcon } from '@angular/material/icon'; +import { Router } from '@angular/router'; import { BackendService } from '../../../services/backend.service'; import { AppService } from '../../../services/app.service'; import { TestGroupsInListDto } from '../../../../../../../api-dto/test-groups/testgroups-in-list.dto'; -import { MatInput } from '@angular/material/input'; interface P { code: string; @@ -38,8 +41,9 @@ interface P { templateUrl: './test-results.component.html', styleUrls: ['./test-results.component.scss'], standalone: true, + providers: [DatePipe], // eslint-disable-next-line max-len - imports: [FormsModule, MatExpansionPanelHeader, MatLabel, MatPaginatorModule, TranslateModule, MatTable, MatCellDef, MatHeaderCellDef, MatHeaderRowDef, MatRowDef, MatCell, MatColumnDef, MatHeaderCell, MatHeaderRow, MatRow, MatSort, MatSortHeader, MatAccordion, MatExpansionPanel, MatExpansionPanelTitle, MatList, MatListItem, MatTooltip, MatInput] + imports: [CommonModule, FormsModule, MatExpansionPanelHeader, MatLabel, MatPaginatorModule, TranslateModule, MatTable, MatCellDef, MatHeaderCellDef, MatHeaderRowDef, MatRowDef, MatCell, MatColumnDef, MatHeaderCell, MatHeaderRow, MatRow, MatSort, MatSortHeader, MatAccordion, MatExpansionPanel, MatExpansionPanelTitle, MatList, MatListItem, MatTooltip, MatInput, MatIcon] }) export class TestResultsComponent implements OnInit { tableSelectionCheckboxes = new SelectionModel(true, []); @@ -53,33 +57,55 @@ export class TestResultsComponent implements OnInit { totalRecords: number = 0; // Gesamtanzahl der Datensätze pageSize: number = 10; // Standardanzahl der Seiten selectedUnit: any; + testPerson: any; + selectedBooklet:any; @ViewChild(MatPaginator) paginator!: MatPaginator; @ViewChild(MatSort) sort!: MatSort; constructor( private backendService: BackendService, - private appService: AppService + private appService: AppService, + private router: Router ) { } ngOnInit(): void { this.createTestResultsList(); - // Setze Paginator und Sortierung - console.log(this.displayedColumns); - console.log(this.dataSource); } onRowClick(row: P): void { - console.log(row); const foundPerson = this.data.find((person: { code: string; }) => person.code === row.code); if (foundPerson && foundPerson.booklets) { this.booklets = foundPerson.booklets; + this.testPerson = foundPerson; } // this.router.navigate(['/detail-view', row.code]); } + replayBooklet(booklet:any) { + this.selectedBooklet = booklet; + } + + replayUnit() { + this.backendService + .createToken(this.appService.selectedWorkspaceId, this.appService.userProfile.id || '', 1) + .subscribe(token => { + const queryParams = { + auth: token + }; + // const page = this.replayComponent.responses?.unit_state?.CURRENT_PAGE_ID; + const url = this.router + .serializeUrl( + this.router.createUrlTree( + [`replay/${this.testPerson.group}@${this.testPerson.code}@${this.selectedBooklet?.id}/${this.selectedUnit.alias}/1`], + { queryParams: queryParams }) + ); + window.open(`#/${url}`, '_blank'); + }); + } + applyFilter(event: Event): void { const filterValue = (event.target as HTMLInputElement).value; this.dataSource.filter = filterValue.trim().toLowerCase(); @@ -90,12 +116,14 @@ export class TestResultsComponent implements OnInit { } } - onListItemClick(unit: any): void { - console.log('Clicked unit:', unit); + onUnitClick(unit: any): void { this.responses = unit.subforms[0].responses; this.logs = this.createUnitHistory(unit); this.selectedUnit = unit; - // Hier kannst du weitere Logik implementieren + } + + setSelectedBooklet(booklet:any) { + this.selectedBooklet = booklet; } calculateDetailedTimeDifferences = (data: { ts: string, key: string, parameter: string }[]) => { @@ -165,7 +193,6 @@ export class TestResultsComponent implements OnInit { this.backendService.getTestResults(this.appService.selectedWorkspaceId) .subscribe(results => { this.data = results; - console.log(results); const mappedResults = results.map(result => ({ code: result.code, group: result.group, @@ -173,13 +200,10 @@ export class TestResultsComponent implements OnInit { uploaded_at: result.uploaded_at })); - // console.log(mappedResults); this.dataSource = new MatTableDataSource(mappedResults); this.totalRecords = mappedResults.length; this.dataSource.paginator = this.paginator; this.dataSource.sort = this.sort; - console.log(this.dataSource, 'this.dataSource'); - console.log(this.displayedColumns); }); } } diff --git a/apps/frontend/src/app/ws-admin/components/unit-results/unit-results.component.ts b/apps/frontend/src/app/ws-admin/components/unit-results/unit-results.component.ts index 2fac9bd2a..9953f868c 100755 --- a/apps/frontend/src/app/ws-admin/components/unit-results/unit-results.component.ts +++ b/apps/frontend/src/app/ws-admin/components/unit-results/unit-results.component.ts @@ -4,14 +4,11 @@ import { import { Component, OnInit, ViewChild } from '@angular/core'; import { MatSort } from '@angular/material/sort'; import { FormsModule } from '@angular/forms'; -import { TranslateModule, TranslateService } from '@ngx-translate/core'; +import { TranslateModule } from '@ngx-translate/core'; import { MatPaginator, MatPaginatorModule } from '@angular/material/paginator'; import { SelectionModel } from '@angular/cdk/collections'; -import { MatSnackBar } from '@angular/material/snack-bar'; - import { BackendService } from '../../../services/backend.service'; import { AppService } from '../../../services/app.service'; - import { TestGroupsInListDto } from '../../../../../../../api-dto/test-groups/testgroups-in-list.dto'; interface P { @@ -46,26 +43,19 @@ export class UnitResultsComponent implements OnInit { constructor( private backendService: BackendService, - private appService: AppService, + private appService: AppService ) { } ngOnInit(): void { this.createTestResultsList(); - // Setze Paginator und Sortierung - - console.log(this.displayedColumns); - console.log(this.dataSource); } onRowClick(row: P): void { - console.log(row); const foundPerson = this.data.find((person: { code: string; }) => person.code === row.code); if (foundPerson && foundPerson.booklets) { this.booklets = foundPerson.booklets; } - - // this.router.navigate(['/detail-view', row.code]); } applyFilter(event: Event): void { @@ -82,7 +72,6 @@ export class UnitResultsComponent implements OnInit { this.backendService.getTestResults(this.appService.selectedWorkspaceId) .subscribe(results => { this.data = results; - console.log(results); const mappedResults = results.map(result => ({ code: result.code, group: result.group, @@ -90,13 +79,10 @@ export class UnitResultsComponent implements OnInit { uploaded_at: result.uploaded_at })); - // console.log(mappedResults); this.dataSource = new MatTableDataSource(mappedResults); this.totalRecords = mappedResults.length; this.dataSource.paginator = this.paginator; this.dataSource.sort = this.sort; - console.log(this.dataSource, 'this.dataSource'); - console.log(this.displayedColumns); }); } } From e0d5d2e37d1cbc67ed68a147a093a8980ee3966e Mon Sep 17 00:00:00 2001 From: jurei733 <67505990+jurei733@users.noreply.github.com> Date: Wed, 12 Mar 2025 12:41:39 +0100 Subject: [PATCH 015/143] Refactor upload results service --- .../services/upload-results.service.ts | 227 ++++++++---------- 1 file changed, 97 insertions(+), 130 deletions(-) diff --git a/apps/backend/src/app/database/services/upload-results.service.ts b/apps/backend/src/app/database/services/upload-results.service.ts index b1d98756d..a647fa315 100644 --- a/apps/backend/src/app/database/services/upload-results.service.ts +++ b/apps/backend/src/app/database/services/upload-results.service.ts @@ -6,6 +6,7 @@ import { Repository } from 'typeorm'; import 'multer'; import { promises as fs, createReadStream, unlinkSync } from 'fs'; import * as csv from 'fast-csv'; +import { Readable } from 'stream'; import { FileIo } from '../../admin/workspace/file-io.interface'; import { Chunk, @@ -23,7 +24,6 @@ import Logs from '../entities/logs.entity'; @Injectable() export class UploadResultsService { private readonly logger = new Logger(UploadResultsService.name); - constructor( @InjectRepository(Persons) private personsRepository: Repository @@ -48,33 +48,37 @@ export class UploadResultsService { if (file.mimetype === 'text/csv') { const randomInteger = Math.floor(Math.random() * 10000); if (file.originalname.includes('logs')) { + console.log('logs'); + const bufferStream = new Readable(); + bufferStream.push(file.buffer); + bufferStream.push(null); // Signalisiert das Ende der Daten + const startTime = performance.now(); const rowData: Log[] = []; - await fs.writeFile(`logs-${randomInteger}.csv`, file.buffer, 'utf-8'); - const stream = createReadStream(`logs-${randomInteger}.csv`); - const csvParserStream = csv.parseStream(stream, { - headers: true, - delimiter: ';', - quote: null - }); - csvParserStream.transform( - (data: Log): Log => ({ - groupname: data.groupname?.replace(/"/g, ''), - loginname: data.loginname?.replace(/"/g, ''), - code: data.code?.replace(/"/g, ''), - bookletname: data.bookletname?.replace(/"/g, ''), - unitname: data.unitname?.replace(/"/g, ''), - timestamp: data.timestamp?.replace(/"/g, ''), - logentry: data.logentry + const csvParserStream = csv + .parseStream(bufferStream, { + headers: true, + delimiter: ';', + quote: null }) + .transform( + (data: Log): Log => ({ + groupname: data.groupname?.replace(/"/g, ''), + loginname: data.loginname?.replace(/"/g, ''), + code: data.code?.replace(/"/g, ''), + bookletname: data.bookletname?.replace(/"/g, ''), + unitname: data.unitname?.replace(/"/g, ''), + timestamp: data.timestamp?.replace(/"/g, ''), + logentry: data.logentry + }) - ) + ) .on('error', error => { - unlinkSync(`logs-${randomInteger}.csv`); this.logger.log(error); }) .on('data', row => rowData.push(row)) .on('end', async () => { - unlinkSync(`logs-${randomInteger}.csv`); + const endTime = performance.now(); + console.log('csv read', `${(endTime - startTime) / 1000}s`); const bookletLogs = []; const unitLogs = []; rowData.forEach(row => { @@ -85,67 +89,57 @@ export class UploadResultsService { } }); this.createPersonList(rowData); - const persons = this.persons.map(person => this.assignBookletLogsToPerson(person, bookletLogs, unitLogs)); - const updateListPromises = []; - for (let i = 0; i < persons.length; i++) { - const person = persons[i]; - updateListPromises.push(this.personsRepository.findOneBy( - { - group: person.group, - code: person.code, - login: person.login - } - ) - .then(p => { - if (p !== null) { - const booklets: TcMergeBooklet[] = p.booklets as TcMergeBooklet[]; - const mappedBooklets = booklets.map(b => - // const mappedUnits = b.units.map(u => { - // unitLogs.forEach(log => { - // if (log.unitname === u.id && log.bookletname === b.id) { - // u.logs.push({ - // ts: log.timestamp, - // key: log.logentry.split('=')[0]?.trim(), - // parameter: log.logentry.split('=')[1]?.trim() - // .replace(/"/g, '') - // }); - // } - // }); - // return u; - // }) - ({ - ...b, - logs: person.booklets.find(pb => pb.id === b.id)?.logs - }) - ); - mappedBooklets.map(booklet => this.assignUnitLogsToBooklet(booklet, unitLogs)); - return { - id: p.id, - ...person, - booklets: mappedBooklets - }; - } + const personTime = performance.now(); + console.log('personTime', `${(personTime - startTime) / 1000}s`); + + const persons = this.persons.map(person => this.assignBookletLogsToPerson(person, bookletLogs)); + const loggedPersonTime = performance.now(); + console.log('loggedPersonTime', `${(loggedPersonTime - startTime) / 1000}s`); + + const keys = persons.map(person => ({ + group: person.group, + code: person.code, + login: person.login + })); - console.log('Person not found in responses'); + const existingPersons = await this.personsRepository.find({ + where: keys + }); + + existingPersons.forEach((p, i) => { + const person = persons[i]; + console.log('person', person); + if (p !== null) { + const booklets: TcMergeBooklet[] = p.booklets as TcMergeBooklet[]; + const mappedBooklets = booklets.map(b => ({ + ...b, + logs: person.booklets.find(pb => pb.id === b.id)?.logs + }) + ); + for (const booklet of mappedBooklets) { + this.assignUnitLogsToBooklet(booklet, unitLogs); } + return { + id: p.id, + ...person, + booklets: mappedBooklets + }; + } - )); - } - const res = await Promise.all(updateListPromises); + console.log('Person not found in responses'); + }); - const chunks = (arr, size) => Array.from( - { length: Math.ceil(arr.length / size) }, - (v, i) => arr.slice(i * size, i * size + size) + const res = existingPersons; + const resolvePromisesTime = performance.now(); + console.log('resolvePromisesTime', `${(resolvePromisesTime - startTime) / 1000}s`); + const chunks = (arr: T[], size: number): T[][] => Array.from({ length: Math.ceil(arr.length / size) }, (_, i) => arr.slice(i * size, i * size + size)); + const chunkedData = chunks(res, 10); + await Promise.all( + chunkedData.map(async chunk => { + await this.personsRepository.upsert(chunk, ['group', 'code', 'login']); + console.log('updated'); + }) ); - - for (let i = 0; i < chunks(res, 10).length; i++) { - const chunk = chunks(res, 10)[i]; - this.personsRepository - .upsert(chunk, ['group', 'code', 'login']).then(() => { - console.log('updated'); - }); - } - //await this.upsertRows(this.personsRepository, rowData); }); } else { console.log('responses'); @@ -177,16 +171,6 @@ export class UploadResultsService { } } - async upsertRows(repository: Repository, rows: Log[]): Promise { - const chunkSize = 1000; // Adjust the chunk size based on your requirements - const chunks = Array.from({ length: Math.ceil(rows.length / chunkSize) }, (v, i) => rows.slice(i * chunkSize, i * chunkSize + chunkSize) - ); - - for (const chunk of chunks) { - await repository.upsert(chunk, ['person_id']); // Replace 'uniqueColumn' with your unique constraint column(s) - } - } - createPersonList(rows: Response[] | Log[] | Logs[]) { const personList : Person[] = []; rows.forEach(row => { @@ -272,55 +256,38 @@ export class UploadResultsService { } assignUnitLogsToBooklet(booklet: TcMergeBooklet, rows: Log[]): TcMergeBooklet { - const units : TcMergeUnit[] = []; + // Map für eindeutigen Zugriff auf Units erstellen + const unitMap = new Map(); + // Direkten Lookup für Units im Booklet vorbereiten + booklet.units.forEach(unit => { + unitMap.set(unit.id, { ...unit, logs: [...unit.logs] }); + }); + + // Logs verarbeiten und zu den Units hinzufügen rows.forEach(row => { - if (booklet?.id === row.bookletname) { - const existingUnit = units.find(u => u?.id === row.unitname); - const existingUnitIndex = units.findIndex(u => u?.id === row.unitname); + if (booklet?.id !== row.bookletname) return; - if (existingUnit) { - existingUnit.logs = [...existingUnit.logs, { - ts: row.timestamp, - key: row.logentry.split('=')[0]?.trim(), - parameter: row.logentry.split('=')[1]?.trim().replace(/"/g, '') - }]; - units[existingUnitIndex] = existingUnit; - } else { - const foundUnit = booklet.units.find(u => u?.id === row.unitname); - if (foundUnit) { - foundUnit.logs.push({ - ts: row.timestamp, - key: row.logentry.split('=')[0]?.trim(), - parameter: row.logentry.split('=')[1]?.trim().replace(/"/g, '') - }); - } - units.push(foundUnit); - } + const logEntryParts = row.logentry.split('='); // Einmal Split durchführen + const log = { + ts: row.timestamp, + key: logEntryParts[0]?.trim(), + parameter: logEntryParts[1]?.trim()?.replace(/"/g, '') + }; + + // Einheit aus der Map holen oder neuen Eintrag hinzufügen + const existingUnit = unitMap.get(row.unitname); + if (existingUnit) { + existingUnit.logs.push(log); + } else { + // Neue Einheit erstellen und in die Map einfügen + const newUnit = { id: row.unitname, logs: [log] } as TcMergeUnit; + unitMap.set(row.unitname, newUnit); } - // - // booklet.units.forEach(unit => { - // if (!units.find(u => row.unitname === u.id)) { - // units.push({ - // ...unit, - // logs: [{ - // ts: row.timestamp, - // key: row.logentry.split('=')[0]?.trim(), - // parameter: row.logentry.split('=')[1]?.trim().replace(/"/g, '') - // }] - // }); - // } else { - // const unitIndex = units.findIndex(u => u.id === row.unitname); - // const logEntry = row.logentry.split('='); - // units[unitIndex].logs.push({ - // ts: row.timestamp, - // key: logEntry[0]?.trim(), - // parameter: logEntry[1]?.trim().replace(/"/g, '') - // }); - // } - // }); }); - booklet.units = units; + + // Map wieder in ein Array umwandeln, um dem ursprünglichen Format zu entsprechen + booklet.units = Array.from(unitMap.values()); return booklet; } From 95a505b9b3f7008770af653d43f8d708688800c3 Mon Sep 17 00:00:00 2001 From: jurei733 <67505990+jurei733@users.noreply.github.com> Date: Wed, 12 Mar 2025 17:00:30 +0100 Subject: [PATCH 016/143] Optimize performance --- .../services/upload-results.service.ts | 336 +++++++++--------- 1 file changed, 171 insertions(+), 165 deletions(-) diff --git a/apps/backend/src/app/database/services/upload-results.service.ts b/apps/backend/src/app/database/services/upload-results.service.ts index a647fa315..90d2edee5 100644 --- a/apps/backend/src/app/database/services/upload-results.service.ts +++ b/apps/backend/src/app/database/services/upload-results.service.ts @@ -4,7 +4,6 @@ import { import { InjectRepository } from '@nestjs/typeorm'; import { Repository } from 'typeorm'; import 'multer'; -import { promises as fs, createReadStream, unlinkSync } from 'fs'; import * as csv from 'fast-csv'; import { Readable } from 'stream'; import { FileIo } from '../../admin/workspace/file-io.interface'; @@ -78,17 +77,20 @@ export class UploadResultsService { .on('data', row => rowData.push(row)) .on('end', async () => { const endTime = performance.now(); - console.log('csv read', `${(endTime - startTime) / 1000}s`); - const bookletLogs = []; - const unitLogs = []; - rowData.forEach(row => { - if (row.unitname === '') { - bookletLogs.push(row); - } else { - unitLogs.push(row); - } - }); + console.log('CSV read duration:', `${(endTime - startTime) / 1000}s`); + + // Trennt die Daten direkt mit `reduce`, um die Logs in einem Schritt zu sortieren. + const { bookletLogs, unitLogs } = rowData.reduce( + (acc, row) => { + row.unitname === '' ? acc.bookletLogs.push(row) : acc.unitLogs.push(row); + return acc; + }, + { bookletLogs: [], unitLogs: [] } + ); + + // Erstellen der Personenliste und Zeitmessung. this.createPersonList(rowData); + const personTime = performance.now(); console.log('personTime', `${(personTime - startTime) / 1000}s`); @@ -108,21 +110,19 @@ export class UploadResultsService { existingPersons.forEach((p, i) => { const person = persons[i]; - console.log('person', person); - if (p !== null) { + console.log('person', person.code); + + if (p) { const booklets: TcMergeBooklet[] = p.booklets as TcMergeBooklet[]; - const mappedBooklets = booklets.map(b => ({ - ...b, - logs: person.booklets.find(pb => pb.id === b.id)?.logs - }) - ); - for (const booklet of mappedBooklets) { - this.assignUnitLogsToBooklet(booklet, unitLogs); - } + const logEnrichedBooklets = booklets.map(b => { + const enriched = this.assignUnitLogsToBooklet(b, unitLogs); + const logs = person.booklets.find(pb => pb.id === b.id)?.logs; + return { enriched, logs }; + }); return { id: p.id, ...person, - booklets: mappedBooklets + booklets: logEnrichedBooklets }; } @@ -130,71 +130,70 @@ export class UploadResultsService { }); const res = existingPersons; - const resolvePromisesTime = performance.now(); - console.log('resolvePromisesTime', `${(resolvePromisesTime - startTime) / 1000}s`); - const chunks = (arr: T[], size: number): T[][] => Array.from({ length: Math.ceil(arr.length / size) }, (_, i) => arr.slice(i * size, i * size + size)); - const chunkedData = chunks(res, 10); - await Promise.all( - chunkedData.map(async chunk => { - await this.personsRepository.upsert(chunk, ['group', 'code', 'login']); - console.log('updated'); - }) - ); + const manipulatedPersonsTime = performance.now(); + console.log('manipulatedPersons', `${(manipulatedPersonsTime - startTime) / 1000}s`); + // const chunks = (arr: T[], size: number): T[][] => Array.from({ length: Math.ceil(arr.length / size) }, (_, i) => arr.slice(i * size, i * size + size)); + // const chunkedData = chunks(res, 10); + // await Promise.all( + // chunkedData.map(async chunk => { + // await this.personsRepository.upsert(chunk, ['group', 'code', 'login']); + // console.log('updated'); + // }) + // ); }); } else { - console.log('responses'); + console.log('Start to import responses. '); const rowData: Response[] = []; - await fs.writeFile(`responses-${randomInteger}.csv`, file.buffer, 'binary'); - const stream = createReadStream(`responses-${randomInteger}.csv`); - csv.parseStream(stream, { headers: true, delimiter: ';' }) + const bufferStream = new Readable(); + bufferStream.push(file.buffer); + bufferStream.push(null); // Signalisiert das Ende der Daten; + csv.parseStream(bufferStream, { headers: true, delimiter: ';' }) .on('error', error => { this.logger.log(error); - unlinkSync(`responses-${randomInteger}.csv`); }) .on('data', row => rowData.push(row)) .on('end', () => { - unlinkSync(`responses-${randomInteger}.csv`); - console.log(rowData[0]); - // const cleanedRows = WorkspaceService.cleanResponses(mappedRowData); - // cleanedRows.forEach(row => filePromises.push( - // this.responsesRepository.upsert(row, ['test_person', 'unit_id']))); - this.createPersonList(rowData); const personList = this.persons.map(person => this.assignBookletsToPerson(person, rowData)) .map(person => this.assignUnitsToBookletAndPerson(person, rowData) ); this.personsRepository.upsert(personList, ['group', 'code', 'login']).then(() => { - console.log('saved'); + console.log(`Saved ${personList.length} test persons`); }); }); } } } - createPersonList(rows: Response[] | Log[] | Logs[]) { - const personList : Person[] = []; + createPersonList(rows: Response[] | Log[] | Logs[]): void { + // Verwendung einer Map für eine effizientere Suche. + const personMap = new Map(); + rows.forEach(row => { - const person = personList - .find(p => p.group === row.groupname && p.login === row.loginname && p.code === row.code); - if (!person) { - personList.push( - { - group: row.groupname, - login: row.loginname, - code: row.code, - booklets: [] - } - ); + const mapKey = `${row.groupname}-${row.loginname}-${row.code}`; + if (!personMap.has(mapKey)) { + personMap.set(mapKey, { + group: row.groupname, + login: row.loginname, + code: row.code, + booklets: [] + }); } }); - this.persons = personList; + + // Konvertiere die Map-Werte in ein Array und weise es zu. + this.persons = Array.from(personMap.values()); } + assignBookletsToPerson(person: Person, rows: Response[]): Person { - const booklets : TcMergeBooklet[] = []; + const bookletIds = new Set(); // Verfolgt eindeutige Booklet-IDs + const booklets: TcMergeBooklet[] = []; + rows.forEach(row => { if (row.groupname === person.group && row.loginname === person.login && row.code === person.code) { - if (!booklets.find(b => b.id === row.bookletname)) { + if (!bookletIds.has(row.bookletname)) { // Prüft effizient, ob `bookletname` bereits hinzugefügt wurde + bookletIds.add(row.bookletname); booklets.push({ id: row.bookletname, logs: [], @@ -204,50 +203,62 @@ export class UploadResultsService { } } }); + person.booklets = booklets; return person; } + assignBookletLogsToPerson(person: Person, rows: Log[]): Person { - const booklets : TcMergeBooklet[] = []; + const booklets: TcMergeBooklet[] = []; + const bookletMap = new Map(); // Map für schnelles Nachschlagen + rows.forEach(row => { if (row.groupname === person.group && row.loginname === person.login && row.code === person.code) { - if (!booklets.find(b => b.id === row.bookletname)) { - const logEntry = row.logentry.split(':', 2); - booklets.push({ - id: row.bookletname, - logs: [{ - ts: row.timestamp, - key: logEntry[0].trim(), - parameter: logEntry[1].trim().replace(/"/g, '') - }], + const { bookletname, timestamp, logentry } = row; + const [logEntryKey, logEntryValueRaw] = logentry.split(':', 2); + const logEntryValue = logEntryValueRaw?.trim().replace(/"/g, ''); + + // Überprüfen, ob das Booklet bereits existiert + let booklet = bookletMap.get(bookletname); + if (!booklet) { + booklet = { + id: bookletname, + logs: [], units: [], sessions: [] - }); - } else { - const bookletIndex = booklets.findIndex(b => b.id === row.bookletname); - const logEntryKey = row.logentry.substring(0, row.logentry.indexOf(':')); - const logEntryValue = row.logentry.substring(row.logentry.indexOf(':') + 3, row.logentry.length - 1).trim().replace(/""/g, '"'); - - if (logEntryKey.trim() === 'LOADCOMPLETE') { - const parsedJSON = JSON.parse(logEntryValue); - const { - browserVersion, browserName, osName, screenSizeWidth, screenSizeHeight, loadTime - } = parsedJSON; - booklets[bookletIndex].sessions.push({ - browser: `${browserName} ${browserVersion}`, - os: `${osName}`, - screen: `${screenSizeWidth} ${screenSizeHeight}`, - ts: row.timestamp, - loadCompleteMS: loadTime - }); - } - booklets[bookletIndex].logs.push({ - ts: row.timestamp, - key: logEntryKey.trim(), - parameter: logEntryValue.trim().replace(/"/g, '') + }; + booklets.push(booklet); + bookletMap.set(bookletname, booklet); + } + + // "LOADCOMPLETE"-Handling + if (logEntryKey.trim() === 'LOADCOMPLETE' && logEntryValue) { + const parsedJSON = JSON.parse(logEntryValue); + const { + browserVersion, + browserName, + osName, + screenSizeWidth, + screenSizeHeight, + loadTime + } = parsedJSON; + + booklet.sessions.push({ + browser: `${browserName} ${browserVersion}`, + os: osName, + screen: `${screenSizeWidth} ${screenSizeHeight}`, + ts: timestamp, + loadCompleteMS: loadTime }); } + + // Log hinzufügen + booklet.logs.push({ + ts: timestamp, + key: logEntryKey.trim(), + parameter: logEntryValue || '' + }); } }); @@ -255,6 +266,7 @@ export class UploadResultsService { return person; } + assignUnitLogsToBooklet(booklet: TcMergeBooklet, rows: Log[]): TcMergeBooklet { // Map für eindeutigen Zugriff auf Units erstellen const unitMap = new Map(); @@ -293,86 +305,80 @@ export class UploadResultsService { assignUnitsToBookletAndPerson(person: Person, rows: Response[]): Person { rows.forEach(row => { - if (row.groupname === person.group && + const matchesPerson = row.groupname === person.group && row.loginname === person.login && - row.code === person.code) { - const booklet = person.booklets.find(b => b.id === row.bookletname); - const responseChunksCleaned = row.responses.replace(/""/g, '"'); - let parsedResponses : Chunk[] = []; - try { - parsedResponses = JSON.parse(responseChunksCleaned); - } catch (e) { - console.log('error', e); - } - const subforms : TcMergeSubForms[] = parsedResponses.filter(chunk => chunk?.id === 'elementCodes').map(chunk => { - let chunkContent : TcMergeResponse[]; + row.code === person.code; + + if (!matchesPerson) return; + + const booklet = person.booklets.find(b => b.id === row.bookletname); + if (!booklet) return; + + // Parse responses + const responseChunksCleaned = row.responses.replace(/""/g, '"'); + let parsedResponses: Chunk[] = []; + try { + parsedResponses = JSON.parse(responseChunksCleaned); + } catch (e) { + console.error('Error parsing responses:', e); + } + + // Extract and map subforms + const subforms: TcMergeSubForms[] = parsedResponses + .filter(chunk => chunk?.id === 'elementCodes') + .map(chunk => { + let chunkContent: TcMergeResponse[] = []; try { chunkContent = JSON.parse(chunk.content); } catch (e) { - console.log('error', e); + console.error('Error parsing chunk content:', e); } - // chunkContent.forEach(cc => { - // try { - // if (cc.value.startsWith('data:application/octet-stream;base64')) { - // console.log('found Geogebra'); - // // const writeStream = fs.createWriteStream('/', { encoding: 'base64' }); - // const hash = crypto.createHash('sha256', { outputLength: 9 }).update(cc.value).digest('base64'); - // fs.writeFile(`GeoGebra/${row.groupname}${row.loginname}${row.code}_${hash}.base64`, cc.value, 'base64', err => { - // console.log('written file'); - // }); - // - // // this.logger.log('hash', hash); - // } - // } catch (e) { - // // console.log('error', e); - // } - // // console.log('response', response); - // }); - return { - id: chunk.id, - responses: chunkContent - }; - }); - const variables = new Set(); - subforms.forEach(subform => { - subform.responses.forEach(response => { - variables.add(response.id); - }); - }); - let parsedLastState = []; - try { - parsedLastState = JSON.parse(row.laststate); - } catch (e) { - console.log('error', e); - } - let laststate: TcMergeLastState[] = []; - if (parsedLastState) { - laststate = Object.entries(parsedLastState).map(ls => ({ key: ls[0], value: ls[1] as string })); - console.log('laststate', laststate); - // - } - person.booklets = person.booklets.map(b => { - if (b.id === booklet.id) { - b.units.push({ - id: row.unitname, - alias: row.unitname, - laststate: laststate, - subforms: subforms, - chunks: [ - { - id: 'elementCodes', - type: parsedResponses[0]?.responseType, - ts: parsedResponses[0]?.ts, - variables: Array.from(variables) - } - ], - logs: [] - }); - } - return b; + return { id: chunk.id, responses: chunkContent }; }); + + // Gather variables from responses + const variables = new Set(); + subforms.forEach(subform => + subform.responses.forEach(response => variables.add(response.id)) + ); + + // Parse laststate + let laststate: TcMergeLastState[] = []; + try { + const parsedLastState = JSON.parse(row.laststate); + laststate = Object.entries(parsedLastState).map(([key, value]) => ({ + key, + value: value as string, + })); + } catch (e) { + console.error('Error parsing last state:', e); } + + // Map and update booklets + person.booklets = person.booklets.map(b => { + if (b.id !== booklet.id) return b; + + const newUnit: TcMergeUnit = { + id: row.unitname, + alias: row.unitname, + laststate, + subforms, + chunks: [ + { + id: 'elementCodes', + type: parsedResponses[0]?.responseType || '', + ts: parsedResponses[0]?.ts || 0, + variables: Array.from(variables), + }, + ], + logs: [], + }; + + b.units.push(newUnit); + return b; + }); }); return person; } + } From 54c6860ee0cc9a873e64e32f3020bc6b8749a089 Mon Sep 17 00:00:00 2001 From: jurei733 <67505990+jurei733@users.noreply.github.com> Date: Sun, 23 Mar 2025 16:36:17 +0100 Subject: [PATCH 017/143] Refactor test result files --- .../admin/workspace/workspace.controller.ts | 12 +++- .../services/upload-results.service.ts | 40 +++++------ .../database/services/workspace.service.ts | 28 ++++++-- .../src/app/services/backend.service.ts | 13 +++- .../test-results/test-results.component.html | 72 ++++++++++--------- .../test-results/test-results.component.scss | 10 +++ .../test-results/test-results.component.ts | 35 +++++---- .../unit-results/unit-results.component.ts | 20 +++--- 8 files changed, 150 insertions(+), 80 deletions(-) diff --git a/apps/backend/src/app/admin/workspace/workspace.controller.ts b/apps/backend/src/app/admin/workspace/workspace.controller.ts index 316fe48b3..604c3ed88 100755 --- a/apps/backend/src/app/admin/workspace/workspace.controller.ts +++ b/apps/backend/src/app/admin/workspace/workspace.controller.ts @@ -116,9 +116,17 @@ export class WorkspaceController { @Get(':workspace_id/test-results') @ApiParam({ name: 'workspace_id', type: Number }) + @ApiOkResponse({ description: 'Test results retrieved successfully.' }) @UseGuards(JwtAuthGuard, WorkspaceGuard) - async findTestResults(@Param('workspace_id') workspace_id: number): Promise { - return this.workspaceService.findTestResults(workspace_id); + async findTestResults( + @Param('workspace_id') workspace_id: number, + @Query('page') page: number = 1, + @Query('limit') limit: number = 10 + ): Promise<{ data: Persons[]; total: number; page: number; limit: number }> { + const [data, total] = await this.workspaceService.findTestResults(workspace_id, { page, limit }); + return { + data, total, page, limit + }; } @Get(':workspace_id/users') diff --git a/apps/backend/src/app/database/services/upload-results.service.ts b/apps/backend/src/app/database/services/upload-results.service.ts index 90d2edee5..b98de57f9 100644 --- a/apps/backend/src/app/database/services/upload-results.service.ts +++ b/apps/backend/src/app/database/services/upload-results.service.ts @@ -88,7 +88,6 @@ export class UploadResultsService { { bookletLogs: [], unitLogs: [] } ); - // Erstellen der Personenliste und Zeitmessung. this.createPersonList(rowData); const personTime = performance.now(); @@ -132,14 +131,14 @@ export class UploadResultsService { const res = existingPersons; const manipulatedPersonsTime = performance.now(); console.log('manipulatedPersons', `${(manipulatedPersonsTime - startTime) / 1000}s`); - // const chunks = (arr: T[], size: number): T[][] => Array.from({ length: Math.ceil(arr.length / size) }, (_, i) => arr.slice(i * size, i * size + size)); - // const chunkedData = chunks(res, 10); - // await Promise.all( - // chunkedData.map(async chunk => { - // await this.personsRepository.upsert(chunk, ['group', 'code', 'login']); - // console.log('updated'); - // }) - // ); + const chunks = (arr: T[], size: number): T[][] => Array.from({ length: Math.ceil(arr.length / size) }, (_, i) => arr.slice(i * size, i * size + size)); + const chunkedData = chunks(res, 10); + await Promise.all( + chunkedData.map(async chunk => { + await this.personsRepository.upsert(chunk, ['group', 'code', 'login']); + console.log('updated'); + }) + ); }); } else { console.log('Start to import responses. '); @@ -185,7 +184,6 @@ export class UploadResultsService { this.persons = Array.from(personMap.values()); } - assignBookletsToPerson(person: Person, rows: Response[]): Person { const bookletIds = new Set(); // Verfolgt eindeutige Booklet-IDs const booklets: TcMergeBooklet[] = []; @@ -208,7 +206,6 @@ export class UploadResultsService { return person; } - assignBookletLogsToPerson(person: Person, rows: Log[]): Person { const booklets: TcMergeBooklet[] = []; const bookletMap = new Map(); // Map für schnelles Nachschlagen @@ -234,7 +231,13 @@ export class UploadResultsService { // "LOADCOMPLETE"-Handling if (logEntryKey.trim() === 'LOADCOMPLETE' && logEntryValue) { - const parsedJSON = JSON.parse(logEntryValue); + let parsedJSON; + try { + parsedJSON = JSON.parse(logEntryValue); + } catch (e) { + console.error('Error parsing JSON:', e); + parsedJSON = {}; + } const { browserVersion, browserName, @@ -266,7 +269,6 @@ export class UploadResultsService { return person; } - assignUnitLogsToBooklet(booklet: TcMergeBooklet, rows: Log[]): TcMergeBooklet { // Map für eindeutigen Zugriff auf Units erstellen const unitMap = new Map(); @@ -338,8 +340,7 @@ export class UploadResultsService { // Gather variables from responses const variables = new Set(); - subforms.forEach(subform => - subform.responses.forEach(response => variables.add(response.id)) + subforms.forEach(subform => subform.responses.forEach(response => variables.add(response.id)) ); // Parse laststate @@ -348,7 +349,7 @@ export class UploadResultsService { const parsedLastState = JSON.parse(row.laststate); laststate = Object.entries(parsedLastState).map(([key, value]) => ({ key, - value: value as string, + value: value as string })); } catch (e) { console.error('Error parsing last state:', e); @@ -368,10 +369,10 @@ export class UploadResultsService { id: 'elementCodes', type: parsedResponses[0]?.responseType || '', ts: parsedResponses[0]?.ts || 0, - variables: Array.from(variables), - }, + variables: Array.from(variables) + } ], - logs: [], + logs: [] }; b.units.push(newUnit); @@ -380,5 +381,4 @@ export class UploadResultsService { }); return person; } - } diff --git a/apps/backend/src/app/database/services/workspace.service.ts b/apps/backend/src/app/database/services/workspace.service.ts index 00163bf28..627ba310f 100755 --- a/apps/backend/src/app/database/services/workspace.service.ts +++ b/apps/backend/src/app/database/services/workspace.service.ts @@ -189,13 +189,33 @@ export class WorkspaceService { }); } - async findTestResults(workspace_id: number): Promise { - this.logger.log('Returning all test results for workspace ', workspace_id); - return this.personsRepository - .find({ + async findTestResults(workspace_id: number, options: { page: number; limit: number }): Promise<[Persons[], number]> { + const { page, limit } = options; + + // Validierungen + if (!workspace_id || workspace_id <= 0) { + throw new Error('Invalid workspace_id provided'); + } + + const MAX_LIMIT = 100; + const validPage = Math.max(1, page); // Minimum 1 + const validLimit = Math.min(Math.max(1, limit), MAX_LIMIT); // Zwischen 1 und MAX_LIMIT + + try { + const [results, total] = await this.personsRepository.findAndCount({ + // where: { workspace_id: workspace_id }, + skip: (validPage - 1) * validLimit, + take: validLimit }); + + return [results, total]; + } catch (error) { + this.logger.error(`Failed to fetch test results for workspace_id ${workspace_id}: ${error.message}`, error.stack); + throw new Error('An error occurred while fetching test results'); + } } + async findUsers(workspace_id: number): Promise { this.logger.log('Returning all users for workspace ', workspace_id); return this.workspaceUsersRepository diff --git a/apps/frontend/src/app/services/backend.service.ts b/apps/frontend/src/app/services/backend.service.ts index a051ad134..54e424e79 100755 --- a/apps/frontend/src/app/services/backend.service.ts +++ b/apps/frontend/src/app/services/backend.service.ts @@ -315,10 +315,19 @@ export class BackendService { { headers: this.authHeader }); } - getTestResults(workspaceId: number): Observable { + getTestResults(workspaceId: number, page: number, limit: number): Observable { + const params = { + page: page.toString(), + limit: limit.toString() + }; + return this.http.get( `${this.serverUrl}admin/workspace/${workspaceId}/test-results/`, - { headers: this.authHeader }); + { + headers: this.authHeader, + params: params + } + ); } authenticate(username:string, password:string, server:string, url:string): Observable { diff --git a/apps/frontend/src/app/ws-admin/components/test-results/test-results.component.html b/apps/frontend/src/app/ws-admin/components/test-results/test-results.component.html index d6f60283c..42861cba7 100755 --- a/apps/frontend/src/app/ws-admin/components/test-results/test-results.component.html +++ b/apps/frontend/src/app/ws-admin/components/test-results/test-results.component.html @@ -6,43 +6,51 @@ Filter - - - - - - {{element.code}} - - - - {{'test_group.test_group' | translate}} - - {{element.group}} - - - - Login - - {{element.login}} - - - - Hinzugefügt am - - {{element.uploaded_at | date: 'dd.MM.yyyy HH:mm'}} - - - - - +
+ +
+ + + + + + {{element.code}} + + + + {{'test_group.test_group' | translate}} + + {{element.group}} + + + + Login + + {{element.login}} + + + + Hinzugefügt am + + {{element.uploaded_at | date: 'dd.MM.yyyy HH:mm'}} + + + + + +
+ [pageIndex]="pageIndex" + [pageSizeOptions]="[20, 50, 100]" + showFirstLastButtons + (page)="onPaginatorChange($event)" + > +
diff --git a/apps/frontend/src/app/ws-admin/components/test-results/test-results.component.scss b/apps/frontend/src/app/ws-admin/components/test-results/test-results.component.scss index dcc1f03ed..864fa46f7 100755 --- a/apps/frontend/src/app/ws-admin/components/test-results/test-results.component.scss +++ b/apps/frontend/src/app/ws-admin/components/test-results/test-results.component.scss @@ -69,3 +69,13 @@ mat-row:hover { border: 2px solid #ddd; /* Optional Rahmen */ border-radius: 8px; /* Runde Ecken, falls benötigt */ } + +.table-container { + max-height: 60%; + overflow-x: auto; /* Ermöglicht horizontales Scrollen */ +} + +mat-header-cell, mat-cell { + white-space: nowrap; /* Verhindert Zeilenumbruch bei langen Inhalten */ +} + diff --git a/apps/frontend/src/app/ws-admin/components/test-results/test-results.component.ts b/apps/frontend/src/app/ws-admin/components/test-results/test-results.component.ts index 724cf8966..43871d620 100755 --- a/apps/frontend/src/app/ws-admin/components/test-results/test-results.component.ts +++ b/apps/frontend/src/app/ws-admin/components/test-results/test-results.component.ts @@ -10,7 +10,7 @@ import { Component, OnInit, ViewChild } from '@angular/core'; import { MatSort, MatSortHeader } from '@angular/material/sort'; import { FormsModule } from '@angular/forms'; import { TranslateModule } from '@ngx-translate/core'; -import { MatPaginator, MatPaginatorModule } from '@angular/material/paginator'; +import { MatPaginator, MatPaginatorModule, PageEvent } from '@angular/material/paginator'; import { SelectionModel } from '@angular/cdk/collections'; import { MatLabel } from '@angular/material/form-field'; import { @@ -27,6 +27,7 @@ import { Router } from '@angular/router'; import { BackendService } from '../../../services/backend.service'; import { AppService } from '../../../services/app.service'; import { TestGroupsInListDto } from '../../../../../../../api-dto/test-groups/testgroups-in-list.dto'; +import { MatProgressSpinner } from '@angular/material/progress-spinner'; interface P { code: string; @@ -43,7 +44,7 @@ interface P { standalone: true, providers: [DatePipe], // eslint-disable-next-line max-len - imports: [CommonModule, FormsModule, MatExpansionPanelHeader, MatLabel, MatPaginatorModule, TranslateModule, MatTable, MatCellDef, MatHeaderCellDef, MatHeaderRowDef, MatRowDef, MatCell, MatColumnDef, MatHeaderCell, MatHeaderRow, MatRow, MatSort, MatSortHeader, MatAccordion, MatExpansionPanel, MatExpansionPanelTitle, MatList, MatListItem, MatTooltip, MatInput, MatIcon] + imports: [CommonModule, FormsModule, MatExpansionPanelHeader, MatLabel, MatPaginatorModule, TranslateModule, MatTable, MatCellDef, MatHeaderCellDef, MatHeaderRowDef, MatRowDef, MatCell, MatColumnDef, MatHeaderCell, MatHeaderRow, MatRow, MatSort, MatSortHeader, MatAccordion, MatExpansionPanel, MatExpansionPanelTitle, MatList, MatListItem, MatTooltip, MatInput, MatIcon, MatProgressSpinner] }) export class TestResultsComponent implements OnInit { tableSelectionCheckboxes = new SelectionModel(true, []); @@ -56,9 +57,12 @@ export class TestResultsComponent implements OnInit { logs: any = []; totalRecords: number = 0; // Gesamtanzahl der Datensätze pageSize: number = 10; // Standardanzahl der Seiten + pageIndex: number = 0; // Aktuelle Seite selectedUnit: any; testPerson: any; selectedBooklet:any; + isLoading: boolean = true; + @ViewChild(MatPaginator) paginator!: MatPaginator; @ViewChild(MatSort) sort!: MatSort; @@ -80,8 +84,6 @@ export class TestResultsComponent implements OnInit { this.booklets = foundPerson.booklets; this.testPerson = foundPerson; } - - // this.router.navigate(['/detail-view', row.code]); } replayBooklet(booklet:any) { @@ -189,20 +191,29 @@ export class TestResultsComponent implements OnInit { } } - createTestResultsList(): void { - this.backendService.getTestResults(this.appService.selectedWorkspaceId) - .subscribe(results => { - this.data = results; - const mappedResults = results.map(result => ({ + onPaginatorChange(event: PageEvent): void { + this.pageSize = event.pageSize; // Aktualisiert die Anzahl der Einträge pro Seite + this.pageIndex = event.pageIndex; // Aktualisiert die aktuelle Seite + this.createTestResultsList(this.pageIndex, this.pageSize); // Läd die Daten neu + } + + createTestResultsList(page: number = 0, limit: number = 10): void { + // Ensure page is non-negative + const validPage = Math.max(0, page); + this.backendService.getTestResults(this.appService.selectedWorkspaceId, validPage, limit) + .subscribe(response => { + this.isLoading = false; + const { data, total } = response; + this.data = data; + const mappedResults = data.map((result: any) => ({ code: result.code, group: result.group, login: result.login, uploaded_at: result.uploaded_at - })); + this.dataSource = new MatTableDataSource(mappedResults); - this.totalRecords = mappedResults.length; - this.dataSource.paginator = this.paginator; + this.totalRecords = total; this.dataSource.sort = this.sort; }); } diff --git a/apps/frontend/src/app/ws-admin/components/unit-results/unit-results.component.ts b/apps/frontend/src/app/ws-admin/components/unit-results/unit-results.component.ts index 9953f868c..376bca8cd 100755 --- a/apps/frontend/src/app/ws-admin/components/unit-results/unit-results.component.ts +++ b/apps/frontend/src/app/ws-admin/components/unit-results/unit-results.component.ts @@ -68,21 +68,25 @@ export class UnitResultsComponent implements OnInit { } } - createTestResultsList(): void { - this.backendService.getTestResults(this.appService.selectedWorkspaceId) - .subscribe(results => { - this.data = results; - const mappedResults = results.map(result => ({ + createTestResultsList(page: number = 0, limit: number = 10): void { + this.backendService.getTestResults(this.appService.selectedWorkspaceId, page, limit) + .subscribe(response => { + // `response` soll die Ergebnisse und die Gesamtanzahl zurückgeben, z. B.: + // { data: [], totalRecords: number }. + const { data, totalRecords } = response; + this.data = data; + + const mappedResults = data.map((result: any) => ({ code: result.code, group: result.group, login: result.login, uploaded_at: result.uploaded_at - })); + this.dataSource = new MatTableDataSource(mappedResults); - this.totalRecords = mappedResults.length; - this.dataSource.paginator = this.paginator; + this.totalRecords = totalRecords; // Gesamtanzahl der Datensätze vom Backend this.dataSource.sort = this.sort; }); } + } From 3e210347aca7906c7b94811e3fdef82aa9ae9a91 Mon Sep 17 00:00:00 2001 From: jurei733 <67505990+jurei733@users.noreply.github.com> Date: Sun, 23 Mar 2025 16:37:24 +0100 Subject: [PATCH 018/143] Create public persons table in 0.4.0 --- .../changelog/coding-box.changelog-0.3.0.sql | 18 ------------------ .../changelog/coding-box.changelog-0.4.0.sql | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+), 18 deletions(-) create mode 100644 database/changelog/coding-box.changelog-0.4.0.sql diff --git a/database/changelog/coding-box.changelog-0.3.0.sql b/database/changelog/coding-box.changelog-0.3.0.sql index 2140f4de3..e092389dd 100644 --- a/database/changelog/coding-box.changelog-0.3.0.sql +++ b/database/changelog/coding-box.changelog-0.3.0.sql @@ -12,21 +12,3 @@ CREATE TABLE "public"."logs" "booklet_id" VARCHAR(100) ); -- rollback DROP TABLE "public"."logs"; - --- changeset jurei733:2 -CREATE TABLE "public"."persons" ( - "id" SERIAL PRIMARY KEY, - "group" VARCHAR(100) NOT NULL, - "login" VARCHAR(100) NOT NULL, - "code" VARCHAR(100) NOT NULL, - "booklets" JSONB, - "workspace_id" INTEGER, - "uploaded_at" TIMESTAMP WITH TIME ZONE DEFAULT now(), - "source" VARCHAR(100) -); --- rollback DROP TABLE "public"."persons"; - --- changeset jurei733:3 -alter table persons add constraint persons_pk - UNIQUE ( group, login, code ); --- rollback alter table persons drop constraint person_id; diff --git a/database/changelog/coding-box.changelog-0.4.0.sql b/database/changelog/coding-box.changelog-0.4.0.sql new file mode 100644 index 000000000..6c8eafd43 --- /dev/null +++ b/database/changelog/coding-box.changelog-0.4.0.sql @@ -0,0 +1,18 @@ +-- liquibase formatted sql + +-- changeset jurei733:1 +CREATE TABLE "public"."persons" ( + "id" SERIAL PRIMARY KEY, + "group" VARCHAR(100) NOT NULL, + "login" VARCHAR(100) NOT NULL, + "code" VARCHAR(100) NOT NULL, + "booklets" JSONB, + "workspace_id" INTEGER, + "uploaded_at" TIMESTAMP WITH TIME ZONE DEFAULT now(), + "source" VARCHAR(100) +); +-- rollback DROP TABLE "public"."persons"; + +-- changeset jurei733:2 +ALTER TABLE "public"."persons" ADD CONSTRAINT person_id UNIQUE ("group",login,code); +-- rollback ALTER TABLE "public"."persons" DROP CONSTRAINT person_id; From b03414963ae3489c78c031d92b964d8294113ef2 Mon Sep 17 00:00:00 2001 From: jurei733 <67505990+jurei733@users.noreply.github.com> Date: Wed, 9 Apr 2025 15:29:00 +0200 Subject: [PATCH 019/143] Cache test-results --- .../admin/workspace/workspace.controller.ts | 2 +- .../database/services/workspace.service.ts | 16 +++++++++--- .../src/app/services/backend.service.ts | 25 ++++++++++++++++++- .../unit-results/unit-results.component.ts | 5 +--- 4 files changed, 39 insertions(+), 9 deletions(-) diff --git a/apps/backend/src/app/admin/workspace/workspace.controller.ts b/apps/backend/src/app/admin/workspace/workspace.controller.ts index 604c3ed88..d78171f92 100755 --- a/apps/backend/src/app/admin/workspace/workspace.controller.ts +++ b/apps/backend/src/app/admin/workspace/workspace.controller.ts @@ -121,7 +121,7 @@ export class WorkspaceController { async findTestResults( @Param('workspace_id') workspace_id: number, @Query('page') page: number = 1, - @Query('limit') limit: number = 10 + @Query('limit') limit: number = 20 ): Promise<{ data: Persons[]; total: number; page: number; limit: number }> { const [data, total] = await this.workspaceService.findTestResults(workspace_id, { page, limit }); return { diff --git a/apps/backend/src/app/database/services/workspace.service.ts b/apps/backend/src/app/database/services/workspace.service.ts index 627ba310f..5b80a41fb 100755 --- a/apps/backend/src/app/database/services/workspace.service.ts +++ b/apps/backend/src/app/database/services/workspace.service.ts @@ -129,6 +129,7 @@ export type TcMergeLastState = { @Injectable() export class WorkspaceService { private readonly logger = new Logger(WorkspaceService.name); + private cache: Map = new Map(); constructor( @InjectRepository(Workspace) @@ -191,8 +192,6 @@ export class WorkspaceService { async findTestResults(workspace_id: number, options: { page: number; limit: number }): Promise<[Persons[], number]> { const { page, limit } = options; - - // Validierungen if (!workspace_id || workspace_id <= 0) { throw new Error('Invalid workspace_id provided'); } @@ -201,6 +200,15 @@ export class WorkspaceService { const validPage = Math.max(1, page); // Minimum 1 const validLimit = Math.min(Math.max(1, limit), MAX_LIMIT); // Zwischen 1 und MAX_LIMIT + const cacheKey = `${workspace_id}-${validPage}-${validLimit}`; + const cacheTTL = 5 * 60 * 1000; + + const cachedData = this.cache.get(cacheKey); + if (cachedData && cachedData.expiry > Date.now()) { + this.logger.log(`Cache hit for workspace_id=${workspace_id}, page=${validPage}, limit=${validLimit}`); + return cachedData.data; + } + try { const [results, total] = await this.personsRepository.findAndCount({ // where: { workspace_id: workspace_id }, @@ -208,6 +216,9 @@ export class WorkspaceService { take: validLimit }); + // save results in cache + this.cache.set(cacheKey, { data: [results, total], expiry: Date.now() + cacheTTL }); + return [results, total]; } catch (error) { this.logger.error(`Failed to fetch test results for workspace_id ${workspace_id}: ${error.message}`, error.stack); @@ -215,7 +226,6 @@ export class WorkspaceService { } } - async findUsers(workspace_id: number): Promise { this.logger.log('Returning all users for workspace ', workspace_id); return this.workspaceUsersRepository diff --git a/apps/frontend/src/app/services/backend.service.ts b/apps/frontend/src/app/services/backend.service.ts index 54e424e79..9a67efd3d 100755 --- a/apps/frontend/src/app/services/backend.service.ts +++ b/apps/frontend/src/app/services/backend.service.ts @@ -1,7 +1,7 @@ import { Injectable, Inject } from '@angular/core'; import { HttpClient } from '@angular/common/http'; import { - catchError, map, Observable, of, switchMap + catchError, map, Observable, of, switchMap, tap } from 'rxjs'; import { CreateUserDto } from '../../../../../api-dto/user/create-user-dto'; import { AppService } from './app.service'; @@ -28,6 +28,8 @@ import Persons from '../../../../backend/src/app/database/entities/persons.entit providedIn: 'root' }) export class BackendService { + private cache = new Map(); // Key-Value-Paar für den Cache + constructor( @Inject('SERVER_URL') private readonly serverUrl: string, private http: HttpClient, public appService: AppService @@ -316,6 +318,11 @@ export class BackendService { } getTestResults(workspaceId: number, page: number, limit: number): Observable { + const cacheKey = `testResults-${workspaceId}-${page}-${limit}`; // unique key for cache data + if (this.cache.has(cacheKey)) { + return of(this.cache.get(cacheKey)); + } + const params = { page: page.toString(), limit: limit.toString() @@ -327,9 +334,25 @@ export class BackendService { headers: this.authHeader, params: params } + ).pipe( + tap(data => { + this.cache.set(cacheKey, data); + }), + catchError(error => { + console.error('Fehler beim Abrufen der Testdaten:', error); + return of(null); + }) ); } + clearCache(key?: string): void { + if (key) { + this.cache.delete(key); + } else { + this.cache.clear(); + } + } + authenticate(username:string, password:string, server:string, url:string): Observable { return this.http .post(`${this.serverUrl}tc_authentication`, { diff --git a/apps/frontend/src/app/ws-admin/components/unit-results/unit-results.component.ts b/apps/frontend/src/app/ws-admin/components/unit-results/unit-results.component.ts index 376bca8cd..7bdac4559 100755 --- a/apps/frontend/src/app/ws-admin/components/unit-results/unit-results.component.ts +++ b/apps/frontend/src/app/ws-admin/components/unit-results/unit-results.component.ts @@ -71,8 +71,6 @@ export class UnitResultsComponent implements OnInit { createTestResultsList(page: number = 0, limit: number = 10): void { this.backendService.getTestResults(this.appService.selectedWorkspaceId, page, limit) .subscribe(response => { - // `response` soll die Ergebnisse und die Gesamtanzahl zurückgeben, z. B.: - // { data: [], totalRecords: number }. const { data, totalRecords } = response; this.data = data; @@ -84,9 +82,8 @@ export class UnitResultsComponent implements OnInit { })); this.dataSource = new MatTableDataSource(mappedResults); - this.totalRecords = totalRecords; // Gesamtanzahl der Datensätze vom Backend + this.totalRecords = totalRecords; this.dataSource.sort = this.sort; }); } - } From 3db4c560a1ec747cce72b24934d988fc0e589b03 Mon Sep 17 00:00:00 2001 From: jurei733 <67505990+jurei733@users.noreply.github.com> Date: Wed, 9 Apr 2025 15:32:09 +0200 Subject: [PATCH 020/143] View test results limited height --- .../test-results/test-results.component.html | 10 +-- .../test-results/test-results.component.scss | 74 ++++++++++++++----- .../test-results/test-results.component.ts | 71 +++++++++--------- 3 files changed, 99 insertions(+), 56 deletions(-) diff --git a/apps/frontend/src/app/ws-admin/components/test-results/test-results.component.html b/apps/frontend/src/app/ws-admin/components/test-results/test-results.component.html index 42861cba7..dae0504f1 100755 --- a/apps/frontend/src/app/ws-admin/components/test-results/test-results.component.html +++ b/apps/frontend/src/app/ws-admin/components/test-results/test-results.component.html @@ -7,7 +7,7 @@
- +
@@ -53,7 +53,7 @@
- + @@ -61,7 +61,7 @@ {{ booklet.id }} - + - + -
+
play_arrow @for ( logBlock of this.logs; track logBlock;) { @for ( log of logBlock; track log;) { diff --git a/apps/frontend/src/app/ws-admin/components/test-results/test-results.component.scss b/apps/frontend/src/app/ws-admin/components/test-results/test-results.component.scss index 864fa46f7..6a8437119 100755 --- a/apps/frontend/src/app/ws-admin/components/test-results/test-results.component.scss +++ b/apps/frontend/src/app/ws-admin/components/test-results/test-results.component.scss @@ -4,8 +4,8 @@ } mat-table{ - max-width:78vh; - max-height:78vh; + max-width:60vh; + height:60vh; overflow-y: scroll; } @@ -15,8 +15,28 @@ coding-box-search-filter { .container{ margin:20px; - max-height: 78vh; - min-height: 78vh; + max-height: 60vh; + min-height: 60vh; +} + +.log-list{ + max-height: 60vh; + overflow-y: auto; +} + +.accordion{ + max-height: 60vh; + overflow-y: auto; +} + +.unit-list{ + max-height: 60vh; + overflow-y: auto; +} + +.var-list{ + max-height: 60vh; + overflow-y: auto; } .upload-buttons{ @@ -40,42 +60,60 @@ mat-row:hover { } .hoverable:hover { - background-color: #e0e0e0; /* Eine leichte Hintergrundfarbe für Hover */ + background-color: #e0e0e0; } .mat-list-item { - padding: 16px; /* Zusätzliche Polsterung für die Liste */ - border-bottom: 1px solid #ddd; /* Optionale Unterkante */ + padding: 16px; + border-bottom: 1px solid #ddd; } .mat-list-item:active { - background-color: #d0d0d0; /* Farbe, wenn das Listenelement aktiv ist */ + background-color: #d0d0d0; } .square-container { display: flex; - flex-wrap: wrap; /* Quadrate umbrechen, falls Platzmangel herrscht */ + flex-wrap: wrap; } .square { - width: 10px; /* Breite des Quadrats */ - height: 10px; /* Höhe des Quadrats (Quadratform) */ + width: 10px; + height: 10px; display: flex; - justify-content: center; /* Zentrieren des Textes horizontal */ - align-items: center; /* Zentrieren des Textes vertikal */ + justify-content: center; + align-items: center; color: #fff; font-weight: bold; - text-transform: capitalize; /* Erster Buchstabe groß */ - border: 2px solid #ddd; /* Optional Rahmen */ - border-radius: 8px; /* Runde Ecken, falls benötigt */ + text-transform: capitalize; + border: 2px solid #ddd; + border-radius: 8px; } .table-container { - max-height: 60%; - overflow-x: auto; /* Ermöglicht horizontales Scrollen */ + position: relative; } mat-header-cell, mat-cell { white-space: nowrap; /* Verhindert Zeilenumbruch bei langen Inhalten */ } +mat-spinner { + position: absolute; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + z-index: 10; +} + +table:not([loading]) { + opacity: 0.5; /* Tabelle kann beim Laden halbtransparent angezeigt werden */ +} + +table { + width: 100%; + opacity: 1; + transition: opacity 0.3s ease-in-out; +} + + diff --git a/apps/frontend/src/app/ws-admin/components/test-results/test-results.component.ts b/apps/frontend/src/app/ws-admin/components/test-results/test-results.component.ts index 43871d620..a78e80853 100755 --- a/apps/frontend/src/app/ws-admin/components/test-results/test-results.component.ts +++ b/apps/frontend/src/app/ws-admin/components/test-results/test-results.component.ts @@ -24,10 +24,10 @@ import { MatInput } from '@angular/material/input'; import { CommonModule, DatePipe } from '@angular/common'; import { MatIcon } from '@angular/material/icon'; import { Router } from '@angular/router'; +import { MatProgressSpinner } from '@angular/material/progress-spinner'; import { BackendService } from '../../../services/backend.service'; import { AppService } from '../../../services/app.service'; import { TestGroupsInListDto } from '../../../../../../../api-dto/test-groups/testgroups-in-list.dto'; -import { MatProgressSpinner } from '@angular/material/progress-spinner'; interface P { code: string; @@ -55,14 +55,15 @@ export class TestResultsComponent implements OnInit { results: any = []; responses: any = []; logs: any = []; - totalRecords: number = 0; // Gesamtanzahl der Datensätze - pageSize: number = 10; // Standardanzahl der Seiten - pageIndex: number = 0; // Aktuelle Seite + totalRecords: number = 0; + pageSize: number = 10; + pageIndex: number = 0; selectedUnit: any; testPerson: any; selectedBooklet:any; isLoading: boolean = true; + private testResultsCache = new Map(); @ViewChild(MatPaginator) paginator!: MatPaginator; @ViewChild(MatSort) sort!: MatSort; @@ -112,7 +113,6 @@ export class TestResultsComponent implements OnInit { const filterValue = (event.target as HTMLInputElement).value; this.dataSource.filter = filterValue.trim().toLowerCase(); - // Paginator auf die erste Seite zurücksetzen if (this.dataSource.paginator) { this.dataSource.paginator.firstPage(); } @@ -146,22 +146,19 @@ export class TestResultsComponent implements OnInit { return results; }; + // eslint-disable-next-line class-methods-use-this groupByPlayerLoading = (array: any[]) => { - const grouped = []; // Array zum Speichern der Blöcke - let currentBlock: any[] = []; // Aktueller Block - + const grouped = []; + let currentBlock: any[] = []; for (const item of array) { if (item.key === 'PLAYER' && item.parameter === 'LOADING') { - // Wenn ein neuer PLAYER_LOADING gefunden wird if (currentBlock.length > 0) { - grouped.push(currentBlock); // Aktuellen Block speichern + grouped.push(currentBlock); } - currentBlock = []; // Neuen Block starten + currentBlock = []; } - currentBlock.push(item); // Aktuelles Item hinzufügen + currentBlock.push(item); } - - // Den letzten Block speichern (falls vorhanden) if (currentBlock.length > 0) { grouped.push(currentBlock); } @@ -170,16 +167,16 @@ export class TestResultsComponent implements OnInit { }; createUnitHistory(unit: { logs: any[]; }): any { - // Aufruf der Funktion return this.groupByPlayerLoading(unit.logs); } - // Konvertiere Timestamp in lesbares Datum + // eslint-disable-next-line class-methods-use-this formatTimestamp(timestamp: string): string { const date = new Date(Number(timestamp)); - return date.toLocaleString(); // z.B. "31.12.2023, 23:59:59" + return date.toLocaleString(); } + // eslint-disable-next-line class-methods-use-this getColor(status: string): string { switch (status) { case 'VALUE_CHANGED': @@ -192,29 +189,37 @@ export class TestResultsComponent implements OnInit { } onPaginatorChange(event: PageEvent): void { - this.pageSize = event.pageSize; // Aktualisiert die Anzahl der Einträge pro Seite - this.pageIndex = event.pageIndex; // Aktualisiert die aktuelle Seite - this.createTestResultsList(this.pageIndex, this.pageSize); // Läd die Daten neu + // Update the number of items displayed per page + this.pageSize = event.pageSize; + + // Update the current page index + this.pageIndex = event.pageIndex; + + // Reload the test results list based on the new page index and size + this.createTestResultsList(this.pageIndex, this.pageSize); } - createTestResultsList(page: number = 0, limit: number = 10): void { - // Ensure page is non-negative + createTestResultsList(page: number = 0, limit: number = 20): void { + // page not negative const validPage = Math.max(0, page); this.backendService.getTestResults(this.appService.selectedWorkspaceId, validPage, limit) .subscribe(response => { this.isLoading = false; const { data, total } = response; - this.data = data; - const mappedResults = data.map((result: any) => ({ - code: result.code, - group: result.group, - login: result.login, - uploaded_at: result.uploaded_at - })); - - this.dataSource = new MatTableDataSource(mappedResults); - this.totalRecords = total; - this.dataSource.sort = this.sort; + this.updateTable(data, total); }); } + + private updateTable(data: any[], total: number): void { + this.data = data; + const mappedResults = data.map((result: any) => ({ + code: result.code, + group: result.group, + login: result.login, + uploaded_at: result.uploaded_at + })); + this.dataSource = new MatTableDataSource(mappedResults); + this.totalRecords = total; + this.dataSource.sort = this.sort; + } } From 38119fd3814cc33844f73700baf35fea00a07f55 Mon Sep 17 00:00:00 2001 From: jurei733 <67505990+jurei733@users.noreply.github.com> Date: Wed, 9 Apr 2025 16:08:37 +0200 Subject: [PATCH 021/143] Add test-persons dto --- .../test-results/testgroups-in-list.dto.ts | 27 +++++++++++++++++++ .../src/app/services/backend.service.ts | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 api-dto/test-results/testgroups-in-list.dto.ts diff --git a/api-dto/test-results/testgroups-in-list.dto.ts b/api-dto/test-results/testgroups-in-list.dto.ts new file mode 100644 index 000000000..9cf5ff3b1 --- /dev/null +++ b/api-dto/test-results/testgroups-in-list.dto.ts @@ -0,0 +1,27 @@ +import { ApiProperty } from '@nestjs/swagger'; + +export class Persons { + @ApiProperty() + id!: number; + + @ApiProperty() + login!: string; + + @ApiProperty() + code!: string; + + @ApiProperty() + group!: string; + + @ApiProperty() + workspace_id!: number; + + @ApiProperty() + uploaded_at!: Date; + + @ApiProperty() + booklets: unknown; + + @ApiProperty() + source!: string; +} diff --git a/apps/frontend/src/app/services/backend.service.ts b/apps/frontend/src/app/services/backend.service.ts index 9a67efd3d..d3827b910 100755 --- a/apps/frontend/src/app/services/backend.service.ts +++ b/apps/frontend/src/app/services/backend.service.ts @@ -22,7 +22,7 @@ import { ResponseDto } from '../../../../../api-dto/responses/response-dto'; import { FilesDto } from '../../../../../api-dto/files/files.dto'; import { UserInListDto } from '../../../../../api-dto/user/user-in-list-dto'; import { UserWorkspaceAccessDto } from '../../../../../api-dto/workspaces/user-workspace-access-dto'; -import Persons from '../../../../backend/src/app/database/entities/persons.entity'; +import { Persons } from '../../../../../api-dto/test-results/testgroups-in-list.dto'; @Injectable({ providedIn: 'root' From 3b4f1d452c3b0916f20d666dbef91516b6a5d42c Mon Sep 17 00:00:00 2001 From: jurei733 <67505990+jurei733@users.noreply.github.com> Date: Thu, 10 Apr 2025 12:39:04 +0200 Subject: [PATCH 022/143] Refactor backend services --- .../src/app/admin/users/users.controller.ts | 2 +- .../database/services/testcenter.service.ts | 268 ++++++----- .../services/upload-results.service.ts | 1 - .../app/database/services/users.service.ts | 241 +++++----- .../database/services/workspace.service.ts | 429 ++++++++++++------ 5 files changed, 571 insertions(+), 370 deletions(-) diff --git a/apps/backend/src/app/admin/users/users.controller.ts b/apps/backend/src/app/admin/users/users.controller.ts index 0e97b84b9..9b8118e16 100755 --- a/apps/backend/src/app/admin/users/users.controller.ts +++ b/apps/backend/src/app/admin/users/users.controller.ts @@ -51,7 +51,7 @@ export class UsersController { @UseGuards(JwtAuthGuard) @ApiBearerAuth() @ApiTags('admin users') - async editUser(@Param('userId') userId:number, @Body() change: UserFullDto): Promise { + async editUser(@Param('userId') userId:number, @Body() change: UserFullDto): Promise { return this.usersService.editUser(userId, change); } diff --git a/apps/backend/src/app/database/services/testcenter.service.ts b/apps/backend/src/app/database/services/testcenter.service.ts index 9d1c8cacf..8a51846d5 100755 --- a/apps/backend/src/app/database/services/testcenter.service.ts +++ b/apps/backend/src/app/database/services/testcenter.service.ts @@ -90,36 +90,30 @@ export class TestcenterService { ) { } - async authenticate(credentials: { username: string, password: string, server:string, url:string }): Promise { - if (!credentials.server && credentials.url !== '') { + async authenticate(credentials: { username: string; password: string; server: string; url: string }): Promise { + const endpoint = credentials.url && !credentials.server ? + `${credentials.url}/api/session/admin` : + `http://iqb-testcenter${credentials.server}.de/api/session/admin`; + + try { const { data } = await firstValueFrom( - this.httpService.put(`${credentials.url}/api/session/admin`, { - name: credentials.username, - password: credentials.password - }, { - httpsAgent: agent - }).pipe( + this.httpService.put(endpoint, + { + name: credentials.username, + password: credentials.password + }, + { + httpsAgent: agent + }).pipe( catchError(error => { - throw new Error(error); + throw new Error(`Authentication failed: ${error?.message || error}`); }) ) ); return data; + } catch (error) { + throw new Error(`Authentication error: ${error.message || 'Unknown error'}`); } - - const { data } = await firstValueFrom( - this.httpService.put(`http://iqb-testcenter${credentials.server}.de/api/session/admin`, { - name: credentials.username, - password: credentials.password - }, { - httpsAgent: agent - }).pipe( - catchError(error => { - throw new Error(error); - }) - ) - ); - return data; } async importWorkspaceFiles( @@ -160,87 +154,110 @@ export class TestcenterService { ); const chunks = createChunks(resultGroupNames, 2); - const unitResponsesPromises = chunks.map(chunk => { - const unitResponsesPromise = this.httpService.axiosRef - .get(url ? `${url}/api/workspace/${tc_workspace}/report/response?dataIds=${chunk.join(',')}` : - `http://iqb-testcenter${server}.de/api/workspace/${tc_workspace}/report/response?dataIds=${chunk.join(',')}`, - { - httpsAgent: agent, - headers: headersRequest - }); - return unitResponsesPromise - .then(callResponse => { - const rows: ResponseDto[] = callResponse.data - .map((unitResponse: UnitResponse) => ({ - test_person: TestcenterService.getTestPersonName(unitResponse), - unit_id: unitResponse.unitname, - responses: unitResponse.responses, - test_group: unitResponse.groupname, - workspace_id: Number(workspace_id), - unit_state: JSON.parse(unitResponse.laststate), - booklet_id: unitResponse.bookletname, - id: undefined, - created_at: undefined - })); - const cleanedRows = WorkspaceService.cleanResponses(rows); - this.responsesRepository.upsert(cleanedRows, ['test_person', 'unit_id']); + const unitResponsesPromises = chunks.map(async chunk => { + const endpoint = url ? + `${url}/api/workspace/${tc_workspace}/report/response?dataIds=${chunk.join(',')}` : + `http://iqb-testcenter${server}.de/api/workspace/${tc_workspace}/report/response?dataIds=${chunk.join(',')}`; + + try { + const { data: rawResponses } = await this.httpService.axiosRef.get(endpoint, { + httpsAgent: agent, + headers: headersRequest }); + + const rows: ResponseDto[] = rawResponses.map((unitResponse: UnitResponse) => ({ + test_person: TestcenterService.getTestPersonName(unitResponse), + unit_id: unitResponse.unitname, + responses: unitResponse.responses, + test_group: unitResponse.groupname, + workspace_id: Number(workspace_id), + unit_state: JSON.parse(unitResponse.laststate), + booklet_id: unitResponse.bookletname, + id: undefined, + created_at: undefined + })); + + const cleanedRows = WorkspaceService.cleanResponses(rows); + await this.responsesRepository.upsert(cleanedRows, ['test_person', 'unit_id']); + } catch (error) { + console.error('Error processing chunk:', error.message || error); + throw error; // Rethrow to handle it globally + } }); - await Promise.all(unitResponsesPromises).then(() => { + + try { + await Promise.all(unitResponsesPromises); result.success = true; - result.responses = report.data.length; - }).catch(() => { + result.responses = report.data.length; // Assuming `report.data` is accessible here + } catch (error) { result.success = false; - }); + } } if (logs === 'true') { - const resultsPromise = this.httpService.axiosRef - .get(url ? `${url}api/workspace/${tc_workspace}/results` : - `http://iqb-testcenter${server}.de/api/workspace/${tc_workspace}/results`, { - httpsAgent: agent, - headers: headersRequest - }); - const report = await resultsPromise.then(res => res); - if (!report) { - throw new Error('could not obtain information about groups from TC'); - } - const resultGroupNames = report.data.map(group => group.groupName); - const createChunks = (a, size) => Array.from( - new Array(Math.ceil(a.length / size)), - (_, i) => a.slice(i * size, i * size + size) - ); + try { + const resultsUrl = url ? + `${url}api/workspace/${tc_workspace}/results` : + `http://iqb-testcenter${server}.de/api/workspace/${tc_workspace}/results`; - const chunks = createChunks(resultGroupNames, 2); - const logsPromises = chunks.map(chunk => { - const logsPromise = this.httpService.axiosRef - .get(url ? `${url}/api/workspace/${tc_workspace}/report/log?dataIds=${chunk.join(',')}` : - `http://iqb-testcenter${server}.de/api/workspace/${tc_workspace}/report/log?dataIds=${chunk.join(',')}`, - { + const report = await this.httpService.axiosRef.get(resultsUrl, { httpsAgent: agent, headers: headersRequest }); - return logsPromise - .then(callResponse => { - const rows:LogsDto[] = callResponse.data - .map((log: Log) => ({ - unit_id: log.unitname, - timestamp: log.timestamp, - test_group: log.groupname, - workspace_id: Number(workspace_id), - log_entry: log.logentry, - booklet_id: log.bookletname, - id: undefined - })); - this.logsRepository.save(rows, { chunk: 50000 }); + + if (!report || !report.data) { + throw new Error('Could not obtain information about groups from TC'); + } + + // extract group name + const resultGroupNames = report.data.map(group => group.groupName); + + // function to create chuncs + const createChunks = (array: T[], size: number): T[][] => Array.from({ length: Math.ceil(array.length / size) }, (_, i) => array.slice(i * size, i * size + size) + ); + + // Split group names into chunks of 2 elements each + const chunks = createChunks(resultGroupNames, 2); + + // Create promises for fetching the logs + const fetchLogsForChunks = async (chunk: string[]): Promise => { + const logsUrl = url ? + `${url}/api/workspace/${tc_workspace}/report/log?dataIds=${chunk.join(',')}` : + `http://iqb-testcenter${server}.de/api/workspace/${tc_workspace}/report/log?dataIds=${chunk.join(',')}`; + + const logsResponse = await this.httpService.axiosRef.get(logsUrl, { + httpsAgent: agent, + headers: headersRequest }); - }); - await Promise.all(logsPromises).then(() => { + + if (logsResponse && logsResponse.data) { + // Convert logs into the desired format + const logsToSave: LogsDto[] = logsResponse.data.map((log: Log) => ({ + unit_id: log.unitname, + timestamp: log.timestamp, + test_group: log.groupname, + workspace_id: Number(workspace_id), + log_entry: log.logentry, + booklet_id: log.bookletname, + id: undefined + })); + + // Save logs (chunk-wise processing for large data volumes) + await this.logsRepository.save(logsToSave, { chunk: 50000 }); + } + }; + + // Retrieve and save all logs for the respective chunks in parallel + await Promise.all(chunks.map(fetchLogsForChunks)); + + // Record success result.success = true; result.logs = report.data.length; - }).catch(() => { + } catch (error) { + // handle errors result.success = false; - }); + console.error('Error fetching logs:', error.message); + } } if (definitions === 'true' || @@ -332,42 +349,47 @@ export class TestcenterService { return `${unitResponse.loginname}@${unitResponse.code}@${unitResponse.bookletname}`; } - async getFile(file:File, server:string, tc_workspace:string, authToken:string, url:string): - Promise<{ - data: File, name: string, type: string, size: number, id: string - }> { + async getFile( + file: File, + server: string, + tcWorkspace: string, + authToken: string, + url?: string + ): Promise<{ + data: File; + name: string; + type: string; + size: number; + id: string; + }> { const headersRequest = { Authtoken: authToken }; - const filePromise = this.httpService.axiosRef - .get(url ? `${url}/api/workspace/${tc_workspace}/file/${file.type}/${file.name}` : - `http://iqb-testcenter${server}.de/api/workspace/${tc_workspace}/file/${file.type}/${file.name}`, - { - httpsAgent: agent, - headers: headersRequest - }); - const fileData = await filePromise.then(res => res.data); - return { - data: fileData, name: file.name, type: file.type, size: file.size, id: file.id - }; - } - // async getPackage(res:File, server:string, tc_workspace:string, authToken:string): Promise { - // const headersRequest = { - // Authtoken: authToken - // }; - // const filePromise = this.httpService.axiosRef - // .get(`http://iqb-testcenter${server}.de/api/workspace/${tc_workspace}/file/${res.type}/${res.name}`, - // { - // httpsAgent: agent, - // headers: headersRequest - // }); - // //const fileData = await filePromise.then(res => res.data); - // //const zip = new AdmZip(Buffer.from(fileData)); - // //const packageFiles = zip.getEntries().map(entry => entry.entryName); - // - // // return { - // // data: fileData, name: res.name, type: res.type, size: res.size, id: res.id - // // }; - // } + // Construct the request URL based on the provided url or fallback to the default URL + const requestUrl = url ? + `${url}/api/workspace/${tcWorkspace}/file/${file.type}/${file.name}` : + `http://iqb-testcenter${server}.de/api/workspace/${tcWorkspace}/file/${file.type}/${file.name}`; + + try { + const response = await this.httpService.axiosRef.get(requestUrl, { + httpsAgent: agent, // Disable SSL validation for HTTPS requests + headers: headersRequest // Add the authorization headers + }); + + const fileData = response.data; + + return { + data: fileData, + name: file.name, + type: file.type, + size: file.size, + id: file.id + }; + } catch (error) { + console.error(`Failed to fetch file: ${file.name}`, error); + + throw new Error('Unable to fetch the file from server.'); + } + } } diff --git a/apps/backend/src/app/database/services/upload-results.service.ts b/apps/backend/src/app/database/services/upload-results.service.ts index b98de57f9..8b94c6f3a 100644 --- a/apps/backend/src/app/database/services/upload-results.service.ts +++ b/apps/backend/src/app/database/services/upload-results.service.ts @@ -33,7 +33,6 @@ export class UploadResultsService { async uploadTestResults(workspace_id: number, originalFiles: FileIo[]): Promise { this.logger.log(`Uploading test results for workspace ${workspace_id}`); - // console.log('originalFiles', originalFiles); const filePromises = []; for (let i = 0; i < originalFiles.length; i++) { const file = originalFiles[i]; diff --git a/apps/backend/src/app/database/services/users.service.ts b/apps/backend/src/app/database/services/users.service.ts index ef6477e9e..8547fc585 100755 --- a/apps/backend/src/app/database/services/users.service.ts +++ b/apps/backend/src/app/database/services/users.service.ts @@ -1,20 +1,17 @@ import { Injectable, Logger, MethodNotAllowedException } from '@nestjs/common'; import { Repository } from 'typeorm'; import { InjectRepository } from '@nestjs/typeorm'; -import { HttpService } from '@nestjs/axios'; import User from '../entities/user.entity'; import { UserFullDto } from '../../../../../../api-dto/user/user-full-dto'; import { CreateUserDto } from '../../../../../../api-dto/user/create-user-dto'; import WorkspaceUser from '../entities/workspace_user.entity'; import { WorkspaceUserInListDto } from '../../../../../../api-dto/user/workspace-user-in-list-dto'; -import { UserWorkspaceAccessDto } from '../../../../../../api-dto/workspaces/user-workspace-access-dto'; import { UserInListDto } from '../../../../../../api-dto/user/user-in-list-dto'; @Injectable() export class UsersService { private readonly logger = new Logger(UsersService.name); constructor( - private httpService: HttpService, @InjectRepository(User) private usersRepository: Repository, @InjectRepository(WorkspaceUser) @@ -23,49 +20,42 @@ export class UsersService { } async findAllFull(workspaceId?: number): Promise { - const validUsers: number[] = []; + const validUsers = new Set(); + if (workspaceId) { + const workspaceUsers = await this.workspaceUserRepository.find({ + where: { workspaceId }, + select: ['userId'] + }); + workspaceUsers.forEach(wsUser => validUsers.add(wsUser.userId)); + } const users: User[] = await this.usersRepository.find({ order: { username: 'ASC' } }); - const returnUsers: UserFullDto[] = []; - users.forEach(user => { - if (!workspaceId || (validUsers.indexOf(user.id) > -1)) { - returnUsers.push({ - id: user.id, - username: user.username, - isAdmin: user.isAdmin - }); - } - }); - return returnUsers; + return users + .filter(user => !workspaceId || validUsers.has(user.id)) // Filter basierend auf der workspaceId, falls vorhanden + .map(user => ({ + id: user.id, + username: user.username, + isAdmin: user.isAdmin + })); } async findAllUsers(workspaceId?: number): Promise { this.logger.log(`Returning users${workspaceId ? ` for workspaceId: ${workspaceId}` : '.'}`); - const validUsers: UserWorkspaceAccessDto[] = []; - if (workspaceId) { - const workspaceUsers: WorkspaceUser[] = await this.workspaceUserRepository - .find({ where: { workspaceId: workspaceId } }); - - workspaceUsers.forEach(wsU => validUsers.push( - { id: wsU.userId, accessLevel: wsU.accessLevel } - )); - } - const users: User[] = await this.usersRepository - .find({ }); - const returnUsers: WorkspaceUserInListDto[] = []; - users.forEach(user => { - if (!workspaceId || - (validUsers.find(validUser => validUser.id === user.id))) { - returnUsers.push({ - id: user.id, - name: user.username, - username: user.username, - accessLevel: validUsers - .find(validUser => validUser.id === user.id)?.accessLevel || 0, - isAdmin: user.isAdmin - }); - } - }); - return returnUsers; + const validUsers = workspaceId ? + await this.workspaceUserRepository.find({ where: { workspaceId } }) : + []; + const validUserMap = new Map( + validUsers.map(wsUser => [wsUser.userId, wsUser.accessLevel]) + ); + const users = await this.usersRepository.find(); + return users + .filter(user => !workspaceId || validUserMap.has(user.id)) + .map(user => ({ + id: user.id, + name: user.username, // Assuming "name" is the same as "username" + username: user.username, + accessLevel: validUserMap.get(user.id) || 0, // Default accessLevel is 0 if not found + isAdmin: user.isAdmin + })); } async patchAllUsers(workspaceId: number, users: UserInListDto[]): Promise { @@ -90,68 +80,98 @@ export class UsersService { } async findUserWorkspaceIds(userId: number): Promise { - this.logger.log(`Returning workspaces for user with id: ${userId}`); - const workspaces = await this.workspaceUserRepository.find({ where: { userId: userId } }); - const workspaceIds = workspaces.map(workspace => workspace.workspaceId); - if (workspaceIds) { - return workspaceIds; - } - return []; + this.logger.log(`Retrieving workspace IDs for user with ID: ${userId}`); + const workspaces = await this.workspaceUserRepository + .find({ where: { userId: userId } }); + const workspaceIds = workspaces + .map(workspace => workspace.workspaceId); + return workspaceIds || []; } - async findUserByIdentity(id: string): Promise { - this.logger.log(`Returning user with id: ${id}`); - const user = await this.usersRepository.findOne({ - where: { identity: id } - }); - if (user) { - return { - id: user.id, - username: user.username, - isAdmin: user.isAdmin - }; + async findUserByIdentity(id: string): Promise { + this.logger.log(`Searching for user with identity: ${id}`); + const user = await this.usersRepository.findOne({ where: { identity: id } }); + + if (!user) { + this.logger.warn(`User with identity ${id} not found.`); + return null; } - return user; + this.logger.log(`Returning user with id: ${user.id}`); + + return { + id: user.id, + username: user.username, + isAdmin: user.isAdmin + } as UserFullDto; } - async editUser(userId:number, change:UserFullDto): Promise { + async editUser(userId: number, change: UserFullDto): Promise { this.logger.log(`Editing user with id: ${userId}`); - await this.usersRepository.save({ id: userId, ...change }); - return []; + const existingUser = await this.usersRepository.findOne({ where: { id: userId } }); + if (!existingUser) { + this.logger.warn(`User with id: ${userId} not found.`); + throw new Error(`User with id: ${userId} not found.`); + } + const updatedUser = await this.usersRepository.save({ id: userId, ...change }); + return updatedUser; } async setUserWorkspaces(userId: number, workspaceIds: number[]): Promise { - this.logger.log(`Setting workspaces for user with id: ${userId}`); - const entries = workspaceIds.map(workspace => ({ userId: userId, workspaceId: workspace })); - const hasRights = this.workspaceUserRepository.find({ where: { userId: userId } }); - if (hasRights) { - await this.workspaceUserRepository.delete({ userId: userId }); + this.logger.log(`Setting workspaces for user with ID: ${userId}`); + const entries = workspaceIds.map(workspaceId => ({ userId, workspaceId })); + try { + const hasRights = await this.workspaceUserRepository.findOne({ where: { userId } }); + if (hasRights) { + this.logger.log(`Existing workspaces found for user ${userId}, deleting...`); + await this.workspaceUserRepository.delete({ userId }); + } + const savedEntries = await this.workspaceUserRepository.save(entries); + + this.logger.log(`Workspaces successfully set for user with ID: ${userId}`); + // Return true if at least one entry was saved + return savedEntries.length > 0; + } catch (error) { + this.logger.error( + `Error setting workspaces for user with ID: ${userId}. Details: ${error.message}`, + error.stack + ); + throw new Error('Failed to set user workspaces'); } - const saved = await this.workspaceUserRepository.save(entries); - return !!saved; } async create(user: CreateUserDto): Promise { - const newUser = this.usersRepository.create(user); - await this.usersRepository.save(newUser); - return newUser.id; + try { + this.logger.log('Creating a new user'); + + const newUser = this.usersRepository.create(user); + const savedUser = await this.usersRepository.save(newUser); + + this.logger.log(`User created successfully with ID: ${savedUser.id}`); + return savedUser.id; + } catch (error) { + this.logger.error('Error creating a new user', error.stack); + throw new Error('Failed to create user'); + } } async createUser(user: CreateUserDto): Promise { - const existingUser: User = await this.usersRepository.findOne({ + const existingUser: User | null = await this.usersRepository.findOne({ where: { username: user.username }, - select: { - username: true, - id: true - } + select: ['id', 'username'] // Fetch only the needed fields for validation }); - this.logger.log(`Creating user with username: ${JSON.stringify(user)}`); + + this.logger.log(`Attempting to create user with username: ${user.username}`); + if (existingUser) { - this.logger.log(`User with username ${user.username} already exists`); + this.logger.warn(`User with username '${user.username}' already exists with ID: ${existingUser.id}`); return existingUser.id; } + const newUser = this.usersRepository.create(user); + await this.usersRepository.save(newUser); + + this.logger.log(`Successfully created user with ID: ${newUser.id}`); return newUser.id; } @@ -179,50 +199,37 @@ export class UsersService { } async createKeycloakUser(keycloakUser: CreateUserDto): Promise { - const existingUser: User = await this.usersRepository.findOne({ - where: { username: keycloakUser.username }, + const { username, identity, issuer } = keycloakUser; + + // Search for an existing user by either username or a combination of identity and issuer + const existingUser = await this.usersRepository.findOne({ + where: [ + { username }, + { identity, issuer } + ], select: { - username: true, - id: true - } - }); - const existingKeycloakUser: User = await this.usersRepository.findOne({ - where: { identity: keycloakUser.identity, issuer: keycloakUser.issuer }, - select: { - username: true, - id: true + id: true, username: true, identity: true, issuer: true } }); + if (existingUser) { - if (keycloakUser.issuer) existingUser.issuer = keycloakUser?.issuer; - if (keycloakUser.identity) existingUser.identity = keycloakUser?.identity; - await this.usersRepository.update( - { id: existingUser.id }, - { - identity: keycloakUser.identity, - issuer: keycloakUser.issuer - } - ); - this.logger.log(`Updating keycloak user with username: ${JSON.stringify(keycloakUser)}`); - return existingKeycloakUser.id; - } - if (existingKeycloakUser) { - if (keycloakUser.issuer) existingKeycloakUser.issuer = keycloakUser?.issuer; - if (keycloakUser.identity) existingKeycloakUser.identity = keycloakUser?.identity; - await this.usersRepository.update( - { id: existingKeycloakUser.id }, - { - identity: keycloakUser.identity, - issuer: keycloakUser.issuer - } - ); - this.logger.log(`Updating keycloak user with username: ${JSON.stringify(keycloakUser)}`); - return existingKeycloakUser.id; - } + // Prepare fields to update if the provided identity or issuer has changed + const updatedFields: Partial = {}; + if (identity && existingUser.identity !== identity) updatedFields.identity = identity; + if (issuer && existingUser.issuer !== issuer) updatedFields.issuer = issuer; + + // Only update the database if there are fields to update + if (Object.keys(updatedFields).length > 0) { + await this.usersRepository.update({ id: existingUser.id }, updatedFields); + this.logger.log(`Updating existing user: ${JSON.stringify({ ...existingUser, ...updatedFields })}`); + } - this.logger.log(`Creating keycloak user with username: ${JSON.stringify(keycloakUser)}`); + return existingUser.id; + } + this.logger.log(`Creating new Keycloak user: ${JSON.stringify(keycloakUser)}`); const newUser = this.usersRepository.create(keycloakUser); await this.usersRepository.save(newUser); + return newUser.id; } } diff --git a/apps/backend/src/app/database/services/workspace.service.ts b/apps/backend/src/app/database/services/workspace.service.ts index 5b80a41fb..25a514d99 100755 --- a/apps/backend/src/app/database/services/workspace.service.ts +++ b/apps/backend/src/app/database/services/workspace.service.ts @@ -7,6 +7,7 @@ import * as cheerio from 'cheerio'; import AdmZip = require('adm-zip'); import * as util from 'util'; import * as fs from 'fs'; +import * as path from 'path'; import Workspace from '../entities/workspace.entity'; import { WorkspaceInListDto } from '../../../../../../api-dto/workspaces/workspace-in-list-dto'; import { WorkspaceFullDto } from '../../../../../../api-dto/workspaces/workspace-full-dto'; @@ -24,10 +25,11 @@ import { ResponseDto } from '../../../../../../api-dto/responses/response-dto'; import Persons from '../entities/persons.entity'; function sanitizePath(filePath: string): string { - if (filePath.indexOf('..') !== -1) { - throw new Error('Invalid file path'); + const normalizedPath = path.normalize(filePath); // System-basiertes Normalisieren + if (normalizedPath.startsWith('..')) { + throw new Error('Invalid file path: Path cannot navigate outside root.'); } - return filePath; + return normalizedPath.replace(/\\/g, '/'); // Einheitliche Darstellung für Pfade } export type Response = { @@ -151,9 +153,12 @@ export class WorkspaceService { } async findAll(): Promise { - this.logger.log('Returning all workspace groups.'); - const workspaces = await this.workspaceRepository.find({}); - return workspaces.map(workspace => ({ id: workspace.id, name: workspace.name })); + this.logger.log('Fetching all workspaces from the repository.'); + const workspaces = await this.workspaceRepository.find({ + select: ['id', 'name'] + }); + this.logger.log(`Found ${workspaces.length} workspaces.`); + return workspaces.map(({ id, name }) => ({ id, name })); } async findAllUserWorkspaces(identity: string): Promise { @@ -195,7 +200,6 @@ export class WorkspaceService { if (!workspace_id || workspace_id <= 0) { throw new Error('Invalid workspace_id provided'); } - const MAX_LIMIT = 100; const validPage = Math.max(1, page); // Minimum 1 const validLimit = Math.min(Math.max(1, limit), MAX_LIMIT); // Zwischen 1 und MAX_LIMIT @@ -240,18 +244,65 @@ export class WorkspaceService { return !!res; } - async findPlayer(workspace_id: number, playerName:string): Promise { - this.logger.log(`Returning ${playerName} for workspace`, workspace_id); - const files = await this.fileUploadRepository - .find({ where: { file_id: playerName.toUpperCase(), workspace_id: workspace_id } }); - return files; + async findPlayer(workspaceId: number, playerName: string): Promise { + if (!workspaceId || typeof workspaceId !== 'number') { + this.logger.error(`Invalid workspaceId provided: ${workspaceId}`); + throw new Error('Invalid workspaceId parameter'); + } + + if (!playerName || typeof playerName !== 'string') { + this.logger.error(`Invalid playerName provided: ${playerName}`); + throw new Error('Invalid playerName parameter'); + } + + this.logger.log(`Attempting to retrieve files for player '${playerName}' in workspace ${workspaceId}`); + + try { + const files = await this.fileUploadRepository.find({ + where: { + file_id: playerName.toUpperCase(), + workspace_id: workspaceId + } + }); + + if (files.length === 0) { + this.logger.warn(`No files found for player '${playerName}' in workspace ${workspaceId}`); + } else { + this.logger.log(`Found ${files.length} file(s) for player '${playerName}' in workspace ${workspaceId}`); + } + + return files; + } catch (error) { + this.logger.error( + `Failed to retrieve files for player '${playerName}' in workspace ${workspaceId}`, + error.stack + ); + throw new Error(`An error occurred while fetching files for player '${playerName}': ${error.message}`); + } } - async findUnitDef(workspace_id:number, unitId: string): Promise { - this.logger.log('Returning unit def for unit', unitId); - const files = await this.fileUploadRepository - .find({ where: { file_id: `${unitId}.VOUD`, workspace_id: workspace_id } }); - return files; + async findUnitDef(workspaceId: number, unitId: string): Promise { + this.logger.log(`Fetching unit definition for unit: ${unitId} in workspace: ${workspaceId}`); + try { + const files = await this.fileUploadRepository.find({ + where: { + file_id: `${unitId}.VOUD`, + workspace_id: workspaceId + } + }); + if (files.length === 0) { + this.logger.warn(`No unit definition found for unit: ${unitId} in workspace: ${workspaceId}`); + } else { + this.logger.log(`Successfully retrieved ${files.length} file(s) for unit: ${unitId}`); + } + return files; + } catch (error) { + this.logger.error( + `Error retrieving unit definition for unit: ${unitId} in workspace: ${workspaceId}`, + error.stack + ); + throw new Error(`Could not retrieve unit definition for unit: ${unitId}`); + } } async findResponse(workspace_id: number, testPerson:string, unitId:string): Promise { @@ -393,9 +444,18 @@ export class WorkspaceService { async create(workspace: CreateWorkspaceDto): Promise { this.logger.log(`Creating workspace with name: ${workspace.name}`); - const newWorkspace = this.workspaceRepository.create(workspace); - await this.workspaceRepository.save(newWorkspace); - return newWorkspace.id; + const newWorkspace = this.workspaceRepository.create({ ...workspace }); + try { + const savedWorkspace = await this.workspaceRepository.save(newWorkspace); + this.logger.log(`Workspace created successfully with ID: ${savedWorkspace.id}`); + return savedWorkspace.id; + } catch (error) { + this.logger.error( + `Failed to create workspace with name: ${workspace.name}`, + error.stack + ); + throw new Error('Workspace creation failed'); + } } async patch(workspaceData: WorkspaceFullDto): Promise { @@ -410,109 +470,179 @@ export class WorkspaceService { } } - async remove(id: number[]): Promise { - this.logger.log(`Deleting workspaces with ids: ${id.join(', ')}`); - await this.workspaceRepository.delete(id); + async remove(ids: number[]): Promise { + if (!ids || ids.length === 0) { + this.logger.warn('No IDs provided for workspace deletion.'); + return; + } + this.logger.log(`Attempting to delete workspaces with IDs: ${ids.join(', ')}`); + try { + const result = await this.workspaceRepository.delete(ids); + + if (result.affected && result.affected > 0) { + this.logger.log(`Successfully deleted ${result.affected} workspace(s) with IDs: ${ids.join(', ')}`); + } else { + this.logger.warn(`No workspaces found with the specified IDs: ${ids.join(', ')}`); + } + } catch (error) { + this.logger.error(`Failed to delete workspaces with IDs: ${ids.join(', ')}. Error: ${error.message}`, error.stack); + throw error; + } } async uploadTestFiles(workspace_id: number, originalFiles: FileIo[]): Promise { this.logger.log(`Uploading test files for workspace ${workspace_id}`); - const filePromises = - originalFiles.map(file => this.handleFile(workspace_id, file)); - const res = await Promise.all(filePromises); - return !!res; + + try { + const results = await Promise.allSettled( + originalFiles.map(file => this.handleFile(workspace_id, file)) + ); + + // Log details of failed uploads for better debugging + const failedFiles = results + .filter(result => result.status === 'rejected') + .map((result, index) => ({ + file: originalFiles[index], + reason: (result as PromiseRejectedResult).reason + })); + + if (failedFiles.length > 0) { + this.logger.warn(`Some files failed to upload for workspace ${workspace_id}:`); + failedFiles.forEach(({ file, reason }) => this.logger.warn(`File: ${JSON.stringify(file)}, Reason: ${reason}`) + ); + } + + // Return 'true' only if all files were uploaded successfully + return failedFiles.length === 0; + } catch (error) { + this.logger.error(`Unexpected error while uploading files for workspace ${workspace_id}:`, error); + throw error; // Re-throw the error to propagate it further + } } handleFile(workspaceId: number, file: FileIo): Array> { const filePromises: Array> = []; - if (file.mimetype === 'text/xml') { - const xmlDocument = cheerio.load(file.buffer.toString(), { - xmlMode: true, - recognizeSelfClosing: true - }); - - const rootTagName = xmlDocument.root().children().first().prop('tagName'); - - if (rootTagName === 'UNIT') { - const fileId = xmlDocument.root().find('Metadata').find(('Id')).text() - .toUpperCase() - .trim(); - - filePromises.push(this.fileUploadRepository.upsert({ - filename: file.originalname, - workspace_id: workspaceId, - file_type: 'Unit', - file_size: file.size, - data: file.buffer.toString(), - file_id: fileId - }, ['file_id'])); - } - } - if (file.mimetype === 'text/html') { - const resourceFileId = WorkspaceService.getPlayerId(file); - filePromises.push(this.fileUploadRepository.upsert({ - filename: file.originalname, - workspace_id: workspaceId, - file_type: 'Resource', - file_size: file.size, - file_id: resourceFileId, - data: file.buffer.toString() - }, ['file_id'])); + switch (file.mimetype) { + case 'text/xml': + filePromises.push(this.handleXmlFile(workspaceId, file)); + break; + case 'text/html': + filePromises.push(this.handleHtmlFile(workspaceId, file)); + break; + case 'application/octet-stream': + filePromises.push(this.handleOctetStreamFile(workspaceId, file)); + break; + case 'application/zip': + case 'application/x-zip-compressed': + case 'application/x-zip': + filePromises.push(...this.handleZipFile(workspaceId, file)); + break; + default: + this.logger.warn(`Unsupported file type: ${file.mimetype}`); } - if (file.mimetype === 'application/octet-stream') { - filePromises.push(this.fileUploadRepository.upsert({ + + return filePromises; + } + + private async handleXmlFile(workspaceId: number, file: FileIo): Promise { + const xmlDocument = cheerio.load(file.buffer.toString(), { + xmlMode: true, + recognizeSelfClosing: true + }); + + const rootTagName = xmlDocument.root().children().first().prop('tagName'); + + if (rootTagName === 'UNIT') { + const fileId = xmlDocument.root().find('Metadata').find('Id').text() + .toUpperCase() + .trim(); + return this.fileUploadRepository.upsert({ filename: file.originalname, workspace_id: workspaceId, - file_id: WorkspaceService.getResourceId(file), // TODO: why? Should be case insensitive - file_type: 'Resource', + file_type: 'Unit', file_size: file.size, - data: file.buffer.toString() - }, ['file_id'])); + data: file.buffer.toString(), + file_id: fileId + }, ['file_id']); } - if (file.mimetype === 'application/zip' || - file.mimetype === 'application/x-zip-compressed' || - file.mimetype === 'application/x-zip' - ) { - const zip = new AdmZip(file.buffer); - if (file.originalname.endsWith('.itcr.zip')) { - const packageFiles = zip.getEntries().map(entry => entry.entryName); - const resourcePackagesPath = './packages'; - const packageName = 'GeoGebra'; - const zipExtractAllToAsync = util.promisify(zip.extractAllToAsync); - filePromises.push(zipExtractAllToAsync(`${resourcePackagesPath}/${packageName}`, true, true) - .then(async () => { - const newResourcePackage = this.resourcePackageRepository.create({ - name: packageName, - elements: packageFiles, - createdAt: new Date() - }); - await this.resourcePackageRepository.save(newResourcePackage); - const sanitizedFileName = sanitizePath(file.originalname); - fs.writeFileSync( - `${resourcePackagesPath}/${packageName}/${sanitizedFileName}`, - file.buffer - ); - return newResourcePackage.id; - }) - ); - } else { - const zipEntries = zip.getEntries(); - zipEntries.forEach(zipEntry => { - const fileContent = zipEntry.getData(); - const sanitizedEntryName = sanitizePath(zipEntry.entryName); - - filePromises.push(Promise.all(this.handleFile(workspaceId, { - fieldname: file.fieldname, - originalname: `${sanitizedEntryName}`, - encoding: file.encoding, - mimetype: WorkspaceService.getMimeType(sanitizedEntryName), - buffer: fileContent, - size: fileContent.length - }))); - }); - } + + return Promise.resolve(); + } + + private async handleHtmlFile(workspaceId: number, file: FileIo): Promise { + const resourceFileId = WorkspaceService.getPlayerId(file); + + return this.fileUploadRepository.upsert({ + filename: file.originalname, + workspace_id: workspaceId, + file_type: 'Resource', + file_size: file.size, + file_id: resourceFileId, + data: file.buffer.toString() + }, ['file_id']); + } + + private async handleOctetStreamFile(workspaceId: number, file: FileIo): Promise { + const resourceId = WorkspaceService.getResourceId(file); + + return this.fileUploadRepository.upsert({ + filename: file.originalname, + workspace_id: workspaceId, + file_id: resourceId, // TODO: Ensure case insensitivity if required + file_type: 'Resource', + file_size: file.size, + data: file.buffer.toString() + }, ['file_id']); + } + + private handleZipFile(workspaceId: number, file: FileIo): Array> { + const filePromises: Array> = []; + const zip = new AdmZip(file.buffer); + + if (file.originalname.endsWith('.itcr.zip')) { + const packageFiles = zip.getEntries().map(entry => entry.entryName); + const resourcePackagesPath = './packages'; + const packageName = 'GeoGebra'; + const zipExtractAllToAsync = util.promisify(zip.extractAllToAsync); + + filePromises.push(zipExtractAllToAsync(`${resourcePackagesPath}/${packageName}`, true, true) + .then(async () => { + const newResourcePackage = this.resourcePackageRepository.create({ + name: packageName, + elements: packageFiles, + createdAt: new Date() + }); + await this.resourcePackageRepository.save(newResourcePackage); + + const sanitizedFileName = sanitizePath(file.originalname); + fs.writeFileSync(`${resourcePackagesPath}/${packageName}/${sanitizedFileName}`, file.buffer); + + return newResourcePackage.id; + })); + } else { + const zipEntries = zip.getEntries(); + zipEntries.forEach(zipEntry => { + const sanitizedEntry = sanitizePath(zipEntry.entryName); + + if (zipEntry.isDirectory) { + // Skip directories as they do not contain data-related content + this.logger.debug(`Skipping directory entry: ${sanitizedEntry}`); + return; + } + + const fileContent = zipEntry.getData(); + filePromises.push(...this.handleFile(workspaceId, { + fieldname: file.fieldname, + originalname: `${sanitizedEntry}`, + encoding: file.encoding, + mimetype: WorkspaceService.getMimeType(sanitizedEntry), + buffer: fileContent, + size: fileContent.length + } as FileIo)); + }); } + return filePromises; } @@ -536,10 +666,15 @@ export class WorkspaceService { }, <{ [key: string]: ResponseDto }>{})); } - async testCenterImport(entries:FileUpload[]): Promise { - const registry = this.fileUploadRepository.create(entries); - const res = await this.fileUploadRepository.upsert(registry, ['file_id']); - return !!res; + async testCenterImport(entries: FileUpload[]): Promise { + try { + const registry = this.fileUploadRepository.create(entries); + await this.fileUploadRepository.upsert(registry, ['file_id']); + return true; + } catch (error) { + this.logger.error('Error during test center import', error); + return false; + } } private static getMimeType(fileName: string): string { @@ -551,36 +686,74 @@ export class WorkspaceService { private static getPlayerId(file: FileIo): string { try { const playerCode = file.buffer.toString(); + + // Load the string into Cheerio for HTML parsing. const playerContent = cheerio.load(playerCode); - const metaData = playerContent.root() - .find('script[type="application/ld+json"]'); - const metadata = JSON.parse(metaData.text()); + + // Search for JSON+LD