From 397de94d20557dc799144c0667f5a64c85db83fe Mon Sep 17 00:00:00 2001 From: Hamza Aburaneh Date: Thu, 16 Jul 2026 08:31:52 -0400 Subject: [PATCH 1/4] chore: add fmt make targets for Java, YAML, and IP whitelist formatting Adds fmt, fmt-yml, and fmt-ips targets to the Makefile. fmt-yml uses prettier with --ignore-path /dev/null to format all YMLs including gitignored files. fmt-ips uses a Python script to reflow the nginx ingress IP whitelist at 120 chars per line. --- Makefile | 23 ++++++++++++++++++++++- scripts/fmt-ips.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 scripts/fmt-ips.py diff --git a/Makefile b/Makefile index dd2d4609..225d78ab 100644 --- a/Makefile +++ b/Makefile @@ -6,13 +6,16 @@ FORMATTER_VERSION = 1.17.0 FORMATTER_JAR = google-java-format-$(FORMATTER_VERSION)-all-deps.jar FORMATTER_URL = https://github.com/google/google-java-format/releases/download/v$(FORMATTER_VERSION)/$(FORMATTER_JAR) JAVA_FILES = $(shell find src -name "*.java") +YML_FILES = $(shell find . -name "*.yml" -not -path "./target/*" -not -path "./node_modules/*") # Default target .PHONY: help help: @echo "Available targets:" + @echo " fmt - Format all Java and YAML files" @echo " format - Format all Java files using Google Java Format" - @echo " check - Check if files need formatting without changing them" + @echo " fmt-yml - Format all YAML files using Prettier" + @echo " fmt-ips - Format IP whitelist in ingress YAML" @echo " clean - Remove the formatter jar" @echo "" @echo "Note: Keep this Makefile locally, don't commit it to your repository" @@ -22,6 +25,24 @@ $(FORMATTER_JAR): @echo "Downloading Google Java Format..." @curl -L $(FORMATTER_URL) -o $(FORMATTER_JAR) +# Format all Java and YAML files +.PHONY: fmt +fmt: format fmt-yml fmt-ips + +# Format YAML files using Prettier +.PHONY: fmt-yml +fmt-yml: + @echo "Formatting YAML files..." + @npx --yes prettier --ignore-path /dev/null --write $(YML_FILES) + @echo "YAML formatting complete!" + +# Format IP whitelist in ingress YAML +.PHONY: fmt-ips +fmt-ips: + @echo "Formatting IP whitelist..." + @python3 scripts/fmt-ips.py + @echo "IP formatting complete!" + # Format all Java files .PHONY: format format: $(FORMATTER_JAR) diff --git a/scripts/fmt-ips.py b/scripts/fmt-ips.py new file mode 100644 index 00000000..1405b361 --- /dev/null +++ b/scripts/fmt-ips.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +import re +import sys + +LINE_WIDTH = 120 +INDENT = " " +FILE = "kubernetes/feedback-viewer-ingress.yml" + +with open(FILE) as f: + content = f.read() + +pattern = r'(nginx\.ingress\.kubernetes\.io/whitelist-source-range:\s*")(.*?)(")' +match = re.search(pattern, content, re.DOTALL) +if not match: + print("Could not find whitelist-source-range annotation") + sys.exit(1) + +ips = [ip.strip() for ip in match.group(2).split(",") if ip.strip()] + +lines = [] +current = [] +current_len = 0 + +for ip in ips: + seg = (", " if current else "") + ip + if current and current_len + len(seg) > LINE_WIDTH: + lines.append(INDENT + ", ".join(current)) + current = [ip] + current_len = len(ip) + else: + current.append(ip) + current_len += len(seg) + +if current: + lines.append(INDENT + ", ".join(current)) + +formatted = "\n" + ",\n".join(lines) + '"' + +new_content = content[: match.start(2)] + formatted + content[match.end(3) :] + +with open(FILE, "w") as f: + f.write(new_content) + +print(f"Formatted {len(ips)} IPs across {len(lines)} lines") From be6860f20823f764756bd9cb9b59ecad0957f50a Mon Sep 17 00:00:00 2001 From: Hamza Aburaneh Date: Thu, 16 Jul 2026 08:31:55 -0400 Subject: [PATCH 2/4] chore: format YAML files with prettier Fixes indentation and quote style in codeql.yml and docker-compose.yml. --- .github/workflows/codeql.yml | 94 ++++++++++++++++++------------------ docker/docker-compose.yml | 2 +- 2 files changed, 48 insertions(+), 48 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index e9194d86..353021b2 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -13,11 +13,11 @@ name: "CodeQL Advanced" on: push: - branches: [ "main" ] + branches: ["main"] pull_request: - branches: [ "main" ] + branches: ["main"] schedule: - - cron: '42 18 * * 2' + - cron: "42 18 * * 2" jobs: analyze: @@ -43,12 +43,12 @@ jobs: fail-fast: false matrix: include: - - language: actions - build-mode: none - - language: java-kotlin - build-mode: none # This mode only analyzes Java. Set this to 'autobuild' or 'manual' to analyze Kotlin too. - - language: javascript-typescript - build-mode: none + - language: actions + build-mode: none + - language: java-kotlin + build-mode: none # This mode only analyzes Java. Set this to 'autobuild' or 'manual' to analyze Kotlin too. + - language: javascript-typescript + build-mode: none # CodeQL supports the following values keywords for 'language': 'actions', 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'rust', 'swift' # Use `c-cpp` to analyze code written in C, C++ or both # Use 'java-kotlin' to analyze code written in Java, Kotlin or both @@ -58,46 +58,46 @@ jobs: # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages steps: - - name: Checkout repository - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@v4 - # Add any setup steps before running the `github/codeql-action/init` action. - # This includes steps like installing compilers or runtimes (`actions/setup-node` - # or others). This is typically only required for manual builds. - # - name: Setup runtime (example) - # uses: actions/setup-example@v1 + # Add any setup steps before running the `github/codeql-action/init` action. + # This includes steps like installing compilers or runtimes (`actions/setup-node` + # or others). This is typically only required for manual builds. + # - name: Setup runtime (example) + # uses: actions/setup-example@v1 - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v4 - with: - languages: ${{ matrix.language }} - build-mode: ${{ matrix.build-mode }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v4 + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. - # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs - queries: security-extended + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + queries: security-extended - # If the analyze step fails for one of the languages you are analyzing with - # "We were unable to automatically build your code", modify the matrix above - # to set the build mode to "manual" for that language. Then modify this step - # to build your code. - # ℹ️ Command-line programs to run using the OS shell. - # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - - name: Run manual build steps - if: matrix.build-mode == 'manual' - shell: bash - run: | - echo 'If you are using a "manual" build mode for one or more of the' \ - 'languages you are analyzing, replace this with the commands to build' \ - 'your code, for example:' - echo ' make bootstrap' - echo ' make release' - exit 1 + # If the analyze step fails for one of the languages you are analyzing with + # "We were unable to automatically build your code", modify the matrix above + # to set the build mode to "manual" for that language. Then modify this step + # to build your code. + # ℹ️ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + - name: Run manual build steps + if: matrix.build-mode == 'manual' + shell: bash + run: | + echo 'If you are using a "manual" build mode for one or more of the' \ + 'languages you are analyzing, replace this with the commands to build' \ + 'your code, for example:' + echo ' make bootstrap' + echo ' make release' + exit 1 - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4 - with: - category: "/language:${{matrix.language}}" + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v4 + with: + category: "/language:${{matrix.language}}" diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 2621a91f..cc4da6e7 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3' +version: "3" services: mongodb: image: mongo:latest From 1e7002682b0c7a643c1d84b0e936f660cffe0001 Mon Sep 17 00:00:00 2001 From: Hamza Aburaneh Date: Thu, 16 Jul 2026 08:33:43 -0400 Subject: [PATCH 3/4] chore: format Java files with google-java-format --- .../java/ca/gc/tbs/config/CacheConfig.java | 15 +- .../java/ca/gc/tbs/config/CachePreloader.java | 98 +- ...CustomizeAuthenticationSuccessHandler.java | 4 +- .../gc/tbs/config/PasswordEncoderConfig.java | 8 +- .../ca/gc/tbs/config/WebSecurityConfig.java | 92 +- .../gc/tbs/controller/BadWordController.java | 275 ++- .../tbs/controller/CustomErrorController.java | 2 - .../tbs/controller/DashboardController.java | 1584 +++++++----- .../gc/tbs/controller/ImportController.java | 1 + .../ca/gc/tbs/controller/LoginController.java | 2 +- .../gc/tbs/controller/ProblemController.java | 2145 +++++++++-------- .../gc/tbs/controller/TopTaskController.java | 355 +-- .../ca/gc/tbs/controller/UserController.java | 49 +- .../java/ca/gc/tbs/domain/BadWordEntry.java | 80 +- src/main/java/ca/gc/tbs/domain/Problem.java | 3 +- .../java/ca/gc/tbs/domain/TopTaskSurvey.java | 1 - .../java/ca/gc/tbs/filter/GcIpFilter.java | 372 ++- .../java/ca/gc/tbs/filter/LanguageFilter.java | 4 +- .../repository/BadWordEntryRepository.java | 32 +- .../repository/CustomTopTaskRepository.java | 2 +- .../CustomTopTaskRepositoryImpl.java | 13 +- .../gc/tbs/repository/ProblemRepository.java | 3 +- .../java/ca/gc/tbs/security/JWTFilter.java | 8 +- src/main/java/ca/gc/tbs/security/JWTUtil.java | 4 +- src/main/java/ca/gc/tbs/service/BadWords.java | 93 +- .../ca/gc/tbs/service/ContentService.java | 51 +- .../ca/gc/tbs/service/DashboardService.java | 133 +- .../gc/tbs/service/ErrorKeywordService.java | 2 +- .../gc/tbs/service/GcIpValidationService.java | 186 +- .../gc/tbs/service/ProblemCacheService.java | 10 +- .../ca/gc/tbs/service/ProblemDateService.java | 43 +- .../java/ca/gc/tbs/service/UserService.java | 281 ++- .../mongodb/datatables/DataTablesInput.java | 4 +- .../mongodb/datatables/DataTablesOutput.java | 4 +- .../datatables/DataTablesRepository.java | 21 +- .../DataTablesRepositoryFactoryBean.java | 10 +- .../datatables/DataTablesRepositoryImpl.java | 59 +- ...omizeAuthenticationSuccessHandlerTest.java | 13 +- .../gc/tbs/controller/AuthControllerTest.java | 16 +- .../UserControllerSecurityTest.java | 17 +- .../tbs/controller/UserControllerXssTest.java | 4 +- .../java/ca/gc/tbs/filter/GcIpFilterTest.java | 165 +- 42 files changed, 3413 insertions(+), 2851 deletions(-) diff --git a/src/main/java/ca/gc/tbs/config/CacheConfig.java b/src/main/java/ca/gc/tbs/config/CacheConfig.java index 9cebeb49..38c8da01 100644 --- a/src/main/java/ca/gc/tbs/config/CacheConfig.java +++ b/src/main/java/ca/gc/tbs/config/CacheConfig.java @@ -16,16 +16,11 @@ public class CacheConfig { @Bean public CacheManager cacheManager() { - CaffeineCacheManager manager = new CaffeineCacheManager( - "problemDates", - "distinctUrls", - "processedProblems", - "dashboardStats", - "gcIpCache" - ); - manager.setCaffeine(Caffeine.newBuilder() - .expireAfterWrite(24, TimeUnit.HOURS) - .maximumSize(1000)); + CaffeineCacheManager manager = + new CaffeineCacheManager( + "problemDates", "distinctUrls", "processedProblems", "dashboardStats", "gcIpCache"); + manager.setCaffeine( + Caffeine.newBuilder().expireAfterWrite(24, TimeUnit.HOURS).maximumSize(1000)); return manager; } } diff --git a/src/main/java/ca/gc/tbs/config/CachePreloader.java b/src/main/java/ca/gc/tbs/config/CachePreloader.java index c7d0b44b..716df03b 100644 --- a/src/main/java/ca/gc/tbs/config/CachePreloader.java +++ b/src/main/java/ca/gc/tbs/config/CachePreloader.java @@ -3,6 +3,9 @@ import ca.gc.tbs.service.DashboardService; import ca.gc.tbs.service.ProblemCacheService; import ca.gc.tbs.service.ProblemDateService; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.stream.Stream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.ApplicationArguments; @@ -11,64 +14,63 @@ import org.springframework.cache.CacheManager; import org.springframework.stereotype.Component; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.stream.Stream; - @Component public class CachePreloader implements ApplicationRunner { - private static final Logger LOGGER = LoggerFactory.getLogger(CachePreloader.class); + private static final Logger LOGGER = LoggerFactory.getLogger(CachePreloader.class); - private final ProblemCacheService problemCacheService; - private final ProblemDateService problemDateService; - private final DashboardService dashboardService; - private final CacheManager cacheManager; + private final ProblemCacheService problemCacheService; + private final ProblemDateService problemDateService; + private final DashboardService dashboardService; + private final CacheManager cacheManager; - public CachePreloader(ProblemCacheService problemCacheService, - ProblemDateService problemDateService, - DashboardService dashboardService, - CacheManager cacheManager) { - this.problemCacheService = problemCacheService; - this.problemDateService = problemDateService; - this.dashboardService = dashboardService; - this.cacheManager = cacheManager; - } + public CachePreloader( + ProblemCacheService problemCacheService, + ProblemDateService problemDateService, + DashboardService dashboardService, + CacheManager cacheManager) { + this.problemCacheService = problemCacheService; + this.problemDateService = problemDateService; + this.dashboardService = dashboardService; + this.cacheManager = cacheManager; + } + + @Override + public void run(ApplicationArguments args) { + LOGGER.info("Preloading caches before web server starts..."); + long start = System.currentTimeMillis(); - @Override - public void run(ApplicationArguments args) { - LOGGER.info("Preloading caches before web server starts..."); - long start = System.currentTimeMillis(); + // 1. Parallel load of base data (lowest tier) + try (var executor = Executors.newVirtualThreadPerTaskExecutor()) { + CompletableFuture problems = + CompletableFuture.runAsync(problemCacheService::getProcessedProblems, executor); + CompletableFuture urls = + CompletableFuture.runAsync( + problemCacheService::getDistinctProcessedUrlsForCache, executor); + CompletableFuture dates = + CompletableFuture.runAsync(problemDateService::getProblemDates, executor); - // 1. Parallel load of base data (lowest tier) - try (var executor = Executors.newVirtualThreadPerTaskExecutor()) { - CompletableFuture problems = - CompletableFuture.runAsync(problemCacheService::getProcessedProblems, executor); - CompletableFuture urls = - CompletableFuture.runAsync(problemCacheService::getDistinctProcessedUrlsForCache, executor); - CompletableFuture dates = - CompletableFuture.runAsync(problemDateService::getProblemDates, executor); + CompletableFuture.allOf(problems, urls, dates).join(); + } + + LOGGER.info("DB caches loaded in {}ms.", System.currentTimeMillis() - start); - CompletableFuture.allOf(problems, urls, dates).join(); - } + // 2. Sequential load of derived data (Dashboard tier) + // This MUST happen after base data is fully joined to avoid triggering redundant re-calculatons + // and ensure the dashboard tier is also warm. + dashboardService.getDashboardStats(); - LOGGER.info("DB caches loaded in {}ms.", System.currentTimeMillis() - start); + LOGGER.info("All caches warm. Total time: {}ms", System.currentTimeMillis() - start); - // 2. Sequential load of derived data (Dashboard tier) - // This MUST happen after base data is fully joined to avoid triggering redundant re-calculatons - // and ensure the dashboard tier is also warm. - dashboardService.getDashboardStats(); - - LOGGER.info("All caches warm. Total time: {}ms", System.currentTimeMillis() - start); - - // Verify caches are populated - Stream.of("processedProblems", "distinctUrls", "dashboardStats", "problemDates").forEach(name -> { - Cache cache = cacheManager.getCache(name); - if (cache != null) { + // Verify caches are populated + Stream.of("processedProblems", "distinctUrls", "dashboardStats", "problemDates") + .forEach( + name -> { + Cache cache = cacheManager.getCache(name); + if (cache != null) { Object val = cache.get("all"); LOGGER.info("Cache '{}' verification: {}", name, val != null ? "HIT ✓" : "MISS ✗"); - } - }); - } + } + }); + } } diff --git a/src/main/java/ca/gc/tbs/config/CustomizeAuthenticationSuccessHandler.java b/src/main/java/ca/gc/tbs/config/CustomizeAuthenticationSuccessHandler.java index 878ff5f7..8764fa3d 100644 --- a/src/main/java/ca/gc/tbs/config/CustomizeAuthenticationSuccessHandler.java +++ b/src/main/java/ca/gc/tbs/config/CustomizeAuthenticationSuccessHandler.java @@ -5,11 +5,11 @@ */ package ca.gc.tbs.config; -import java.io.IOException; -import java.net.URI; import jakarta.servlet.ServletException; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.net.URI; import org.springframework.security.core.Authentication; import org.springframework.security.core.GrantedAuthority; import org.springframework.security.web.authentication.SimpleUrlAuthenticationSuccessHandler; diff --git a/src/main/java/ca/gc/tbs/config/PasswordEncoderConfig.java b/src/main/java/ca/gc/tbs/config/PasswordEncoderConfig.java index 7c08e5ab..a7838ebe 100644 --- a/src/main/java/ca/gc/tbs/config/PasswordEncoderConfig.java +++ b/src/main/java/ca/gc/tbs/config/PasswordEncoderConfig.java @@ -8,8 +8,8 @@ @Configuration public class PasswordEncoderConfig { - @Bean - public PasswordEncoder bCryptPasswordEncoder() { - return new BCryptPasswordEncoder(); - } + @Bean + public PasswordEncoder bCryptPasswordEncoder() { + return new BCryptPasswordEncoder(); + } } diff --git a/src/main/java/ca/gc/tbs/config/WebSecurityConfig.java b/src/main/java/ca/gc/tbs/config/WebSecurityConfig.java index 45ee754b..29977ed4 100644 --- a/src/main/java/ca/gc/tbs/config/WebSecurityConfig.java +++ b/src/main/java/ca/gc/tbs/config/WebSecurityConfig.java @@ -1,12 +1,12 @@ package ca.gc.tbs.config; +import ca.gc.tbs.security.JWTFilter; import jakarta.servlet.http.HttpServletResponse; - import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.security.config.annotation.method.configuration.EnableMethodSecurity; import org.springframework.security.authentication.AuthenticationManager; import org.springframework.security.config.annotation.authentication.configuration.AuthenticationConfiguration; +import org.springframework.security.config.annotation.method.configuration.EnableMethodSecurity; import org.springframework.security.config.annotation.web.builders.HttpSecurity; import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; import org.springframework.security.config.annotation.web.configuration.WebSecurityCustomizer; @@ -15,8 +15,6 @@ import org.springframework.security.web.util.matcher.AntPathRequestMatcher; import org.thymeleaf.extras.springsecurity6.dialect.SpringSecurityDialect; -import ca.gc.tbs.security.JWTFilter; - @Configuration @EnableWebSecurity @EnableMethodSecurity @@ -35,42 +33,50 @@ public WebSecurityConfig( @Bean public SecurityFilterChain filterChain(HttpSecurity http) throws Exception { - http - .csrf(csrf -> csrf - .ignoringRequestMatchers("/authenticate")) - .authorizeHttpRequests(auth -> auth - .requestMatchers("/createApiUser").hasAuthority("ADMIN") - .requestMatchers("/authenticate").permitAll() - .requestMatchers("/actuator/health").permitAll() - .requestMatchers("/api/user/**").hasRole("USER") - .requestMatchers("/", "/checkExists", "/error", "/login", "/signup", "/success").permitAll() - .requestMatchers("/u/**").hasAnyAuthority("ADMIN") - .requestMatchers("/keywords/**").hasAnyAuthority("ADMIN") - .requestMatchers("/python/**", "/reports/**", "/dashboard/**").hasAnyAuthority("USER", "ADMIN") - .anyRequest().authenticated() - ) - .formLogin(form -> form - .loginPage("/login") - .permitAll() - .successHandler(customizeAuthenticationSuccessHandler) - .failureUrl("/login?error=true") - .usernameParameter("email") - .passwordParameter("password") - ) - .logout(logout -> logout - .logoutRequestMatcher(new AntPathRequestMatcher("/logout")) - .logoutSuccessUrl("/login?logout=true") - ) - .exceptionHandling(ex -> ex - .authenticationEntryPoint( - (request, response, authException) -> { - if ("XMLHttpRequest".equals(request.getHeader("X-Requested-With"))) { - response.sendError(HttpServletResponse.SC_UNAUTHORIZED, "Unauthorized"); - } else { - response.sendRedirect("/login"); - } - }) - ) + http.csrf(csrf -> csrf.ignoringRequestMatchers("/authenticate")) + .authorizeHttpRequests( + auth -> + auth.requestMatchers("/createApiUser") + .hasAuthority("ADMIN") + .requestMatchers("/authenticate") + .permitAll() + .requestMatchers("/actuator/health") + .permitAll() + .requestMatchers("/api/user/**") + .hasRole("USER") + .requestMatchers("/", "/checkExists", "/error", "/login", "/signup", "/success") + .permitAll() + .requestMatchers("/u/**") + .hasAnyAuthority("ADMIN") + .requestMatchers("/keywords/**") + .hasAnyAuthority("ADMIN") + .requestMatchers("/python/**", "/reports/**", "/dashboard/**") + .hasAnyAuthority("USER", "ADMIN") + .anyRequest() + .authenticated()) + .formLogin( + form -> + form.loginPage("/login") + .permitAll() + .successHandler(customizeAuthenticationSuccessHandler) + .failureUrl("/login?error=true") + .usernameParameter("email") + .passwordParameter("password")) + .logout( + logout -> + logout + .logoutRequestMatcher(new AntPathRequestMatcher("/logout")) + .logoutSuccessUrl("/login?logout=true")) + .exceptionHandling( + ex -> + ex.authenticationEntryPoint( + (request, response, authException) -> { + if ("XMLHttpRequest".equals(request.getHeader("X-Requested-With"))) { + response.sendError(HttpServletResponse.SC_UNAUTHORIZED, "Unauthorized"); + } else { + response.sendRedirect("/login"); + } + })) .addFilterBefore(jwtFilter, UsernamePasswordAuthenticationFilter.class); return http.build(); @@ -78,8 +84,9 @@ public SecurityFilterChain filterChain(HttpSecurity http) throws Exception { @Bean public WebSecurityCustomizer webSecurityCustomizer() { - return web -> web.ignoring() - .requestMatchers("/resources/**", "/static/**", "/css/**", "/js/**", "/images/**"); + return web -> + web.ignoring() + .requestMatchers("/resources/**", "/static/**", "/css/**", "/js/**", "/images/**"); } @Bean @@ -92,5 +99,4 @@ public AuthenticationManager authenticationManager(AuthenticationConfiguration c throws Exception { return config.getAuthenticationManager(); } - } diff --git a/src/main/java/ca/gc/tbs/controller/BadWordController.java b/src/main/java/ca/gc/tbs/controller/BadWordController.java index c1e023bf..883111fe 100644 --- a/src/main/java/ca/gc/tbs/controller/BadWordController.java +++ b/src/main/java/ca/gc/tbs/controller/BadWordController.java @@ -1,14 +1,15 @@ package ca.gc.tbs.controller; +import ca.gc.tbs.domain.BadWordEntry; +import ca.gc.tbs.repository.BadWordEntryRepository; +import ca.gc.tbs.service.BadWords; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import java.io.BufferedReader; import java.io.InputStreamReader; import java.io.PrintWriter; import java.nio.charset.StandardCharsets; import java.util.List; - -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; - import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVPrinter; @@ -25,14 +26,9 @@ import org.springframework.web.multipart.MultipartFile; import org.springframework.web.servlet.ModelAndView; -import ca.gc.tbs.domain.BadWordEntry; -import ca.gc.tbs.repository.BadWordEntryRepository; -import ca.gc.tbs.service.BadWords; - /** - * Controller for managing BadWordEntry entities. - * Provides CRUD operations and CSV import/export functionality. - * All routes are restricted to ADMIN users only. + * Controller for managing BadWordEntry entities. Provides CRUD operations and CSV import/export + * functionality. All routes are restricted to ADMIN users only. */ @Controller @PreAuthorize("hasAuthority('ADMIN')") @@ -45,7 +41,7 @@ public class BadWordController { /** * Displays the keywords management page. - * + * * @param request HttpServletRequest to get session language * @return ModelAndView with the keywords table data */ @@ -63,7 +59,7 @@ public ModelAndView index(HttpServletRequest request) { /** * Generates HTML table rows for all badword entries. - * + * * @param lang The language for labels ("en" or "fr") * @return HTML string containing table rows */ @@ -79,72 +75,92 @@ private String getData(String lang) { String language = entry.getLanguage(); boolean active = entry.getActive(); - String langLabel = switch (language) { - case "en" -> "EN"; - case "fr" -> "FR"; - default -> "BOTH"; - }; - - String langClass = switch (language) { - case "en" -> "tag-en"; - case "fr" -> "tag-fr"; - default -> "tag-both"; - }; - - String typeLabel = (lang.equals("fr")) ? switch (type) { - case "profanity" -> "VULGAIRE"; - case "threat" -> "MENACE"; - case "allowed" -> "AUTORISÉ"; - case "error" -> "ERREUR"; - default -> type.toUpperCase(); - } : type.toUpperCase(); - - String activeLabel = lang.equals("en") ? (active ? "ACTIVE" : "INACTIVE") : (active ? "ACTIF" : "INACTIF"); + String langLabel = + switch (language) { + case "en" -> "EN"; + case "fr" -> "FR"; + default -> "BOTH"; + }; + + String langClass = + switch (language) { + case "en" -> "tag-en"; + case "fr" -> "tag-fr"; + default -> "tag-both"; + }; + + String typeLabel = + (lang.equals("fr")) + ? switch (type) { + case "profanity" -> "VULGAIRE"; + case "threat" -> "MENACE"; + case "allowed" -> "AUTORISÉ"; + case "error" -> "ERREUR"; + default -> type.toUpperCase(); + } + : type.toUpperCase(); + + String activeLabel = + lang.equals("en") ? (active ? "ACTIVE" : "INACTIVE") : (active ? "ACTIF" : "INACTIF"); String activeClass = active ? "tag-active" : "tag-inactive"; String actionButtons; if (lang.equals("en")) { - String toggleBtn = active - ? """ + String toggleBtn = + active + ? """ """ - .formatted(id) - : """ + .formatted(id) + : """ """ - .formatted(id); + .formatted(id); - actionButtons = """ + actionButtons = + """ %s """ - .formatted(id, id, word, language, type, active, toggleBtn, id); + .formatted(id, id, word, language, type, active, toggleBtn, id); } else { - String toggleBtn = active - ? """ + String toggleBtn = + active + ? """ """ - .formatted(id) - : """ + .formatted(id) + : """ """ - .formatted(id); + .formatted(id); - actionButtons = """ + actionButtons = + """ %s """ - .formatted(id, id, word, language, type, active, toggleBtn, id); + .formatted(id, id, word, language, type, active, toggleBtn, id); } - builder.append(""" + builder.append( + """ %s %s %s %s %s - """.formatted(word, langClass, langLabel, type, typeLabel, activeClass, activeLabel, actionButtons)); + """ + .formatted( + word, + langClass, + langLabel, + type, + typeLabel, + activeClass, + activeLabel, + actionButtons)); } } catch (Exception e) { LOG.error("Error generating keywords table data", e); @@ -154,7 +170,7 @@ private String getData(String lang) { /** * Creates a new badword entry. - * + * * @param word The word text * @param language The language ("en", "fr", or "both") * @param type The type ("profanity", "threat", "allowed", "error") @@ -178,30 +194,35 @@ private String getData(String lang) { if (!isValidType(type)) { return "Error: Invalid type. Must be 'profanity', 'threat', 'allowed', or 'error'"; } - + String normalizedWord = word.trim().toLowerCase(); - + // Check for duplicates - BadWordEntry existing = repository.findByWordAndLanguageAndType(normalizedWord, language, type); + BadWordEntry existing = + repository.findByWordAndLanguageAndType(normalizedWord, language, type); if (existing != null) { return "Error: This word already exists for the specified language and type"; } - + // Create new entry BadWordEntry entry = new BadWordEntry(); entry.setWord(normalizedWord); entry.setLanguage(language); entry.setType(type); entry.setActive(active); - + repository.save(entry); - + // Reload the in-memory cache badWordsService.reload(); - - LOG.info("Created new badword entry: word={}, language={}, type={}, active={}", - normalizedWord, language, type, active); - + + LOG.info( + "Created new badword entry: word={}, language={}, type={}, active={}", + normalizedWord, + language, + type, + active); + return "Success"; } catch (Exception e) { LOG.error("Error creating badword entry", e); @@ -211,7 +232,7 @@ private String getData(String lang) { /** * Updates an existing badword entry. - * + * * @param id The entry ID * @param word The new word text (optional) * @param language The new language (optional) @@ -231,9 +252,9 @@ private String getData(String lang) { if (entry == null) { return "Error: Entry not found"; } - + boolean changed = false; - + // Update word if provided if (word != null && !word.trim().isEmpty()) { String normalizedWord = word.trim().toLowerCase(); @@ -241,7 +262,8 @@ private String getData(String lang) { // Check for duplicates with new word String checkLanguage = language != null ? language : entry.getLanguage(); String checkType = type != null ? type : entry.getType(); - BadWordEntry existing = repository.findByWordAndLanguageAndType(normalizedWord, checkLanguage, checkType); + BadWordEntry existing = + repository.findByWordAndLanguageAndType(normalizedWord, checkLanguage, checkType); if (existing != null && !existing.getId().equals(id)) { return "Error: This word already exists for the specified language and type"; } @@ -249,7 +271,7 @@ private String getData(String lang) { changed = true; } } - + // Update language if provided if (language != null && !language.isEmpty()) { if (!isValidLanguage(language)) { @@ -257,8 +279,8 @@ private String getData(String lang) { } if (!language.equals(entry.getLanguage())) { // Check for duplicates with new language - BadWordEntry existing = repository.findByWordAndLanguageAndType( - entry.getWord(), language, entry.getType()); + BadWordEntry existing = + repository.findByWordAndLanguageAndType(entry.getWord(), language, entry.getType()); if (existing != null && !existing.getId().equals(id)) { return "Error: This word already exists for the specified language and type"; } @@ -266,7 +288,7 @@ private String getData(String lang) { changed = true; } } - + // Update type if provided if (type != null && !type.isEmpty()) { if (!isValidType(type)) { @@ -274,8 +296,8 @@ private String getData(String lang) { } if (!type.equals(entry.getType())) { // Check for duplicates with new type - BadWordEntry existing = repository.findByWordAndLanguageAndType( - entry.getWord(), entry.getLanguage(), type); + BadWordEntry existing = + repository.findByWordAndLanguageAndType(entry.getWord(), entry.getLanguage(), type); if (existing != null && !existing.getId().equals(id)) { return "Error: This word already exists for the specified language and type"; } @@ -283,19 +305,19 @@ private String getData(String lang) { changed = true; } } - + // Update active status if provided if (active != null && active != entry.getActive()) { entry.setActive(active); changed = true; } - + if (changed) { repository.save(entry); badWordsService.reload(); LOG.info("Updated badword entry: id={}", id); } - + return "Success"; } catch (Exception e) { LOG.error("Error updating badword entry", e); @@ -305,7 +327,7 @@ private String getData(String lang) { /** * Deletes a badword entry. - * + * * @param id The entry ID * @return Success or error message */ @@ -315,10 +337,10 @@ private String getData(String lang) { if (!repository.existsById(id)) { return "Error: Entry not found"; } - + repository.deleteById(id); badWordsService.reload(); - + LOG.info("Deleted badword entry: id={}", id); return "Success"; } catch (Exception e) { @@ -329,7 +351,7 @@ private String getData(String lang) { /** * Exports all badword entries to CSV. - * + * * @param response HttpServletResponse to write CSV data */ @GetMapping(value = "/keywords/export") @@ -337,25 +359,22 @@ public void exportCsv(HttpServletResponse response) { try { response.setContentType("text/csv; charset=UTF-8"); response.setHeader("Content-Disposition", "attachment; filename=\"keywords.csv\""); - + List entries = repository.findAll(); - + try (PrintWriter writer = response.getWriter(); - CSVPrinter csvPrinter = new CSVPrinter(writer, CSVFormat.DEFAULT - .withHeader("word", "language", "type", "active"))) { - + CSVPrinter csvPrinter = + new CSVPrinter( + writer, CSVFormat.DEFAULT.withHeader("word", "language", "type", "active"))) { + // Write data for (BadWordEntry entry : entries) { csvPrinter.printRecord( - entry.getWord(), - entry.getLanguage(), - entry.getType(), - entry.getActive().toString() - ); + entry.getWord(), entry.getLanguage(), entry.getType(), entry.getActive().toString()); } csvPrinter.flush(); } - + LOG.info("Exported {} keyword entries to CSV", entries.size()); } catch (Exception e) { LOG.error("Error exporting keywords to CSV", e); @@ -363,9 +382,8 @@ public void exportCsv(HttpServletResponse response) { } /** - * Imports badword entries from CSV file. - * Expected CSV format: word,language,type,active - * + * Imports badword entries from CSV file. Expected CSV format: word,language,type,active + * * @param file The uploaded CSV file * @return Success or error message with import statistics */ @@ -375,25 +393,26 @@ public void exportCsv(HttpServletResponse response) { if (file.isEmpty()) { return "Error: No file uploaded"; } - + int imported = 0; int skipped = 0; int errors = 0; - - try (BufferedReader reader = new BufferedReader( - new InputStreamReader(file.getInputStream(), StandardCharsets.UTF_8)); - CSVParser csvParser = new CSVParser(reader, CSVFormat.DEFAULT - .withFirstRecordAsHeader() - .withIgnoreHeaderCase() - .withTrim())) { - + + try (BufferedReader reader = + new BufferedReader( + new InputStreamReader(file.getInputStream(), StandardCharsets.UTF_8)); + CSVParser csvParser = + new CSVParser( + reader, + CSVFormat.DEFAULT.withFirstRecordAsHeader().withIgnoreHeaderCase().withTrim())) { + for (CSVRecord record : csvParser) { try { String word = record.get("word"); String language = record.get("language"); String type = record.get("type"); String activeStr = record.get("active"); - + // Validate if (word == null || word.trim().isEmpty()) { errors++; @@ -407,68 +426,68 @@ public void exportCsv(HttpServletResponse response) { errors++; continue; } - + String normalizedWord = word.trim().toLowerCase(); Boolean active = activeStr != null ? Boolean.parseBoolean(activeStr) : true; - + // Check for duplicates - BadWordEntry existing = repository.findByWordAndLanguageAndType( - normalizedWord, language, type); - + BadWordEntry existing = + repository.findByWordAndLanguageAndType(normalizedWord, language, type); + if (existing != null) { skipped++; continue; } - + // Create entry BadWordEntry entry = new BadWordEntry(); entry.setWord(normalizedWord); entry.setLanguage(language); entry.setType(type); entry.setActive(active); - + repository.save(entry); imported++; - + } catch (Exception e) { LOG.error("Error importing CSV row: {}", record, e); errors++; } } } - + // Reload cache after import badWordsService.reload(); - - LOG.info("CSV import completed: imported={}, skipped={}, errors={}", imported, skipped, errors); - - return String.format("Import completed: %d imported, %d skipped (duplicates), %d errors", + + LOG.info( + "CSV import completed: imported={}, skipped={}, errors={}", imported, skipped, errors); + + return String.format( + "Import completed: %d imported, %d skipped (duplicates), %d errors", imported, skipped, errors); - + } catch (Exception e) { LOG.error("Error importing CSV file", e); return "Error: Failed to import CSV file."; } } - /** - * Validates language value. - */ + /** Validates language value. */ private boolean isValidLanguage(String language) { - return language != null && (language.equals("en") || language.equals("fr") || language.equals("both")); + return language != null + && (language.equals("en") || language.equals("fr") || language.equals("both")); } - /** - * Validates type value. - */ + /** Validates type value. */ private boolean isValidType(String type) { - return type != null && (type.equals("profanity") || type.equals("threat") - || type.equals("allowed") || type.equals("error")); + return type != null + && (type.equals("profanity") + || type.equals("threat") + || type.equals("allowed") + || type.equals("error")); } - /** - * Escapes HTML special characters to prevent XSS. - */ + /** Escapes HTML special characters to prevent XSS. */ private String escapeHtml(String text) { if (text == null) { return ""; diff --git a/src/main/java/ca/gc/tbs/controller/CustomErrorController.java b/src/main/java/ca/gc/tbs/controller/CustomErrorController.java index fac4f79b..a300c33c 100644 --- a/src/main/java/ca/gc/tbs/controller/CustomErrorController.java +++ b/src/main/java/ca/gc/tbs/controller/CustomErrorController.java @@ -1,7 +1,6 @@ package ca.gc.tbs.controller; import jakarta.servlet.http.HttpServletRequest; - import org.springframework.boot.web.servlet.error.ErrorController; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.RequestMapping; @@ -18,5 +17,4 @@ public String handleError(HttpServletRequest request) { } return "error_" + lang; } - } diff --git a/src/main/java/ca/gc/tbs/controller/DashboardController.java b/src/main/java/ca/gc/tbs/controller/DashboardController.java index 6e369fb6..801bca48 100644 --- a/src/main/java/ca/gc/tbs/controller/DashboardController.java +++ b/src/main/java/ca/gc/tbs/controller/DashboardController.java @@ -1,5 +1,16 @@ package ca.gc.tbs.controller; +import ca.gc.tbs.domain.Problem; +import ca.gc.tbs.service.DashboardService; +import ca.gc.tbs.service.ErrorKeywordService; +import ca.gc.tbs.service.ProblemDateService; +import ca.gc.tbs.service.UserService; +import jakarta.servlet.ServletOutputStream; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import jakarta.validation.Valid; +import java.io.IOException; +import java.io.Writer; import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.ArrayList; @@ -8,606 +19,1075 @@ import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; -import java.util.List; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; -import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; - -import jakarta.servlet.ServletOutputStream; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; -import jakarta.validation.Valid; -import java.io.IOException; -import java.io.Writer; - +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.xssf.streaming.SXSSFSheet; +import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.bson.Document; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.data.domain.Sort; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.aggregation.Aggregation; -import org.springframework.data.mongodb.core.aggregation.AggregationResults; -import org.springframework.data.mongodb.core.aggregation.GroupOperation; -import org.springframework.data.mongodb.core.aggregation.MatchOperation; -import org.springframework.data.mongodb.core.aggregation.SortOperation; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.datatables.DataTablesInput; import org.springframework.data.mongodb.datatables.DataTablesOutput; - -import org.apache.poi.ss.usermodel.Row; -import org.apache.poi.ss.usermodel.Sheet; -import org.apache.poi.xssf.streaming.SXSSFSheet; -import org.apache.poi.xssf.streaming.SXSSFWorkbook; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.servlet.ModelAndView; -import ca.gc.tbs.domain.Problem; -import ca.gc.tbs.service.DashboardService; -import ca.gc.tbs.service.ErrorKeywordService; -import ca.gc.tbs.service.ProblemDateService; -import ca.gc.tbs.service.UserService; - @Controller public class DashboardController { - private static final Logger LOG = LoggerFactory.getLogger(DashboardController.class); - - private final ProblemDateService problemDateService; - private final DashboardService dashboardService; - private final UserService userService; - private final ErrorKeywordService errorKeywordService; - private final MongoTemplate mongoTemplate; - - public DashboardController( - ProblemDateService problemDateService, - DashboardService dashboardService, - UserService userService, - ErrorKeywordService errorKeywordService, - MongoTemplate mongoTemplate) { - this.problemDateService = problemDateService; - this.dashboardService = dashboardService; - this.userService = userService; - this.errorKeywordService = errorKeywordService; - this.mongoTemplate = mongoTemplate; - } - - private static final Map> institutionMappings = new HashMap<>(); - private static final Map> sectionMappings = new HashMap<>(); - - static { - // Initialize section mappings - sectionMappings.put("disability", Arrays.asList("disability", "disability benefits")); - sectionMappings.put("news", Arrays.asList("news")); - - // Initialize institution mappings (kept as in the original file) - institutionMappings.put("AAFC", Arrays.asList("AAFC", "AAC", "AGRICULTURE AND AGRI-FOOD CANADA", "AGRICULTURE ET AGROALIMENTAIRE CANADA", "AAFC/AAC")); - institutionMappings.put("ACOA", Arrays.asList("ACOA", "APECA", "ATLANTIC CANADA OPPORTUNITIES AGENCY", "AGENCE DE PROMOTION ÉCONOMIQUE DU CANADA ATLANTIQUE", "ACOA/APECA")); - institutionMappings.put("ATSSC", Arrays.asList("ATSSC", "SCDATA", "ADMINISTRATIVE TRIBUNALS SUPPORT SERVICE OF CANADA", "SERVICE CANADIEN D’APPUI AUX TRIBUNAUX ADMINISTRATIFS", "ATSSC/SCDATA")); - institutionMappings.put("CANNOR", Arrays.asList("CANNOR", "RNCAN", "CANADIAN NORTHERN ECONOMIC DEVELOPMENT AGENCY", "AGENCE CANADIENNE DE DÉVELOPPEMENT ÉCONOMIQUE DU NORD", "CANNOR/RNCAN")); - institutionMappings.put("CATSA", Arrays.asList("CATSA", "ACSTA", "CANADIAN AIR TRANSPORT SECURITY AUTHORITY", "ADMINISTRATION CANADIENNE DE LA SÛRETÉ DU TRANSPORT AÉRIEN", "CATSA/ACSTA")); - institutionMappings.put("CBSA", Arrays.asList("CBSA", "ASFC", "CANADA BORDER SERVICES AGENCY", "AGENCE DES SERVICES FRONTALIERS DU CANADA", "CBSA/ASFC")); - institutionMappings.put("CCG", Arrays.asList("CCG", "GCC", "CANADIAN COAST GUARD", "GARDE CÔTIÈRE CANADIENNE", "CCG/GCC")); - institutionMappings.put("CER", Arrays.asList("CER", "REC", "CANADA ENERGY REGULATOR", "RÉGIE DE L'ÉNERGIE DU CANADA", "CER/REC")); - institutionMappings.put("CFIA", Arrays.asList("CFIA", "ACIA", "CANADIAN FOOD INSPECTION AGENCY", "AGENCE CANADIENNE D’INSPECTION DES ALIMENTS", "CFIA/ACIA")); - institutionMappings.put("CIHR", Arrays.asList("CIHR", "IRSC", "CANADIAN INSTITUTES OF HEALTH RESEARCH", "INSTITUTS DE RECHERCHE EN SANTÉ DU CANADA", "CIHR/IRSC")); - institutionMappings.put("CIPO", Arrays.asList("CIPO", "OPIC", "CANADIAN INTELLECTUAL PROPERTY OFFICE", "OFFICE DE LA PROPRIÉTÉ INTELLECTUELLE DU CANADA", "CIPO/OPIC")); - institutionMappings.put("CIRNAC", Arrays.asList("CIRNAC", "RCAANC", "CROWN-INDIGENOUS RELATIONS AND NORTHERN AFFAIRS CANADA", "RELATIONS COURONNE-AUTOCHTONES ET AFFAIRES DU NORD CANADA", "CIRNAC/RCAANC")); - institutionMappings.put("CRA", Arrays.asList("CRA", "ARC", "CANADA REVENUE AGENCY", "AGENCE DU REVENU DU CANADA", "CRA/ARC")); - institutionMappings.put("CRTC", Arrays.asList("CRTC", "CRTC", "CANADIAN RADIO-TELEVISION AND TELECOMMUNICATIONS COMMISSION", "CONSEIL DE LA RADIODIFFUSION ET DES TÉLÉCOMMUNICATIONS CANADIENNES")); - institutionMappings.put("CSA", Arrays.asList("CSA", "ASC", "CANADIAN SPACE AGENCY", "AGENCE SPATIALE CANADIENNE", "CSA/ASC")); - institutionMappings.put("CSC", Arrays.asList("CSC", "SCC", "CORRECTIONAL SERVICE CANADA", "SERVICE CORRECTIONNEL CANADA", "CSC/SCC")); - institutionMappings.put("CSE", Arrays.asList("CSE", "CST", "COMMUNICATIONS SECURITY ESTABLISHMENT", "CENTRE DE LA SÉCURITÉ DES TÉLÉCOMMUNICATIONS", "CSE/CST")); - institutionMappings.put("CSEC", Arrays.asList("CSEC", "CSTC", "COMMUNICATIONS SECURITY ESTABLISHMENT CANADA", "CENTRE DE LA SÉCURITÉ DES TÉLÉCOMMUNICATIONS CANADA", "CSEC/CSTC")); - institutionMappings.put("CSPS", Arrays.asList("CSPS", "EFPC", "CANADA SCHOOL OF PUBLIC SERVICE", "ÉCOLE DE LA FONCTION PUBLIQUE DU CANADA", "CSPS/EFPC")); - institutionMappings.put("DFO", Arrays.asList("DFO", "MPO", "FISHERIES AND OCEANS CANADA", "PÊCHES ET OCÉANS CANADA", "DFO/MPO", "GOVERNMENT OF CANADA, FISHERIES AND OCEANS CANADA, COMMUNICATIONS BRANCH")); - institutionMappings.put("DND", Arrays.asList("DND", "MDN", "NATIONAL DEFENCE", "DÉFENSE NATIONALE", "DND/MDN")); - institutionMappings.put("ECCC", Arrays.asList("ECCC", "ENVIRONMENT AND CLIMATE CHANGE CANADA", "ENVIRONNEMENT ET CHANGEMENT CLIMATIQUE CANADA", "ECCC")); - institutionMappings.put("ESDC", Arrays.asList("ESDC", "EDSC", "EMPLOYMENT AND SOCIAL DEVELOPMENT CANADA", "EMPLOI ET DÉVELOPPEMENT SOCIAL CANADA", "ESDC/EDSC", "EMPLOI ET DÉVÉLOPPEMENT SOCIALE CANADA")); - institutionMappings.put("FCAC", Arrays.asList("FCAC", "ACFC", "FINANCIAL CONSUMER AGENCY OF CANADA", "AGENCE DE LA CONSOMMATION EN MATIÈRE FINANCIÈRE DU CANADA", "FCAC/ACFC")); - institutionMappings.put("FIN", Arrays.asList("FIN", "FIN", "FINANCE CANADA", "MINISTÈRE DES FINANCES CANADA", "DEPARTMENT OF FINANCE CANADA", "GOVERNMENT OF CANADA, DEPARTMENT OF FINANCE", "MINISTÈRE DES FINANCES", "FIN")); - institutionMappings.put("GAC", Arrays.asList("GAC", "AMC", "GLOBAL AFFAIRS CANADA", "AFFAIRES MONDIALES CANADA", "GAC/AMC")); - institutionMappings.put("HC", Arrays.asList("HC", "SC", "HEALTH CANADA", "SANTÉ CANADA", "HC/SC")); - institutionMappings.put("HICC", Arrays.asList("HICC", "LICC", "HOUSING, INFRASTRUCTURE AND COMMUNITIES CANADA", "LOGEMENT, INFRASTRUCTURES ET COLLECTIVITÉS CANADA", "HICC/LICC")); - institutionMappings.put("INFC", Arrays.asList("INFC", "INFC", "INFRASTRUCTURE CANADA", "INFRASTRUCTURE CANADA", "INFC / INFC")); - institutionMappings.put("IOGC", Arrays.asList("IOGC", "BPGI", "INDIAN OIL AND GAS CANADA", "BUREAU DU PÉTROLE ET DU GAZ DES INDIENS", "IOGC/BPGI")); - institutionMappings.put("IRCC", Arrays.asList("IRCC", "IRCC", "IMMIGRATION, REFUGEES AND CITIZENSHIP CANADA", "IMMIGRATION, RÉFUGIÉS ET CITOYENNETÉ CANADA")); - institutionMappings.put("ISC", Arrays.asList("ISC", "SAC", "INDIGENOUS SERVICES CANADA", "SERVICES AUX AUTOCHTONES CANADA", "ISC/SAC")); - institutionMappings.put("ISED", Arrays.asList("ISED", "ISDE", "INNOVATION, SCIENCE AND ECONOMIC DEVELOPMENT CANADA", "INNOVATION, SCIENCES ET DÉVELOPPEMENT ÉCONOMIQUE CANADA", "ISED/ISDE")); - institutionMappings.put("JUS", Arrays.asList("JUS", "JUSTICE CANADA", "MINISTÈRE DE LA JUSTICE CANADA", "JUS")); - institutionMappings.put("LAC", Arrays.asList("LAC", "BAC", "LIBRARY AND ARCHIVES CANADA", "BIBLIOTHÈQUE ET ARCHIVES CANADA", "LAC/BAC")); - institutionMappings.put("NFB", Arrays.asList("NFB", "ONF", "NATIONAL FILM BOARD", "OFFICE NATIONAL DU FILM", "NFB/ONF")); - institutionMappings.put("NRC", Arrays.asList("NRC", "CNRC", "NATIONAL RESEARCH COUNCIL", "CONSEIL NATIONAL DE RECHERCHES CANADA", "NRC/CNRC")); - institutionMappings.put("NRCAN", Arrays.asList("NRCAN", "RNCAN", "NATURAL RESOURCES CANADA", "RESSOURCES NATURELLES CANADA", "NRCAN/RNCAN")); - institutionMappings.put("NSERC", Arrays.asList("NSERC", "CRSNG", "NATURAL SCIENCES AND ENGINEERING RESEARCH CANADA", "CONSEIL DE RECHERCHES EN SCIENCES NATURELLES ET EN GÉNIE DU CANADA", "NSERC/CRSNG")); - institutionMappings.put("OMBDNDCAF", Arrays.asList("OMBDNDCAF", "OMBMDNFAC", "DND/CAF OMBUDSMAN", "OMBUDSMAN DU MDN/FAC", "OFFICE OF THE NATIONAL DEFENCE AND CANADIAN ARMED FORCES OMBUDSMAN", "BUREAU DE L'OMBUDSMAN DE LA DÉFENSE NATIONALE ET DES FORCES ARMÉES CANADIENNES", "OMBDNDCAF/OMBMDNFAC")); - institutionMappings.put("OSB", Arrays.asList("OSB", "BSF", "SUPERINTENDENT OF BANKRUPTCY CANADA", "BUREAU DU SURINTENDANT DES FAILLITES CANADA", "OSB/BSF")); - institutionMappings.put("PBC", Arrays.asList("PBC", "CLCC", "PAROLE BOARD OF CANADA", "COMMISSION DES LIBÉRATIONS CONDITIONNELLES DU CANADA", "PBC/CLCC")); - institutionMappings.put("PC", Arrays.asList("PC", "PC", "PARCS CANADA", "PARKS CANADA")); - institutionMappings.put("PCH", Arrays.asList("PCH", "PCH", "CANADIAN HERITAGE", "PATRIMOINE CANADIEN")); - institutionMappings.put("PCO", Arrays.asList("PCO", "BCP", "PRIVY COUNCIL OFFICE", "BUREAU DU CONSEIL PRIVÉ", "PCO/BCP")); - institutionMappings.put("PHAC", Arrays.asList("PHAC", "ASPC", "PUBLIC HEALTH AGENCY OF CANADA", "AGENCE DE LA SÉAUTÉ PUBLIQUE DU CANADA", "PHAC/ASPC")); - institutionMappings.put("PS", Arrays.asList("PS", "SP", "PUBLIC SAFETY CANADA", "SÉCURITÉ PUBLIQUE CANADA", "PS/SP")); - institutionMappings.put("PSC", Arrays.asList("PSC", "CFP", "PUBLIC SERVICE COMMISSION OF CANADA", "COMMISSION DE LA FONCTION PUBLIQUE DU CANADA", "PSC/CFP")); - institutionMappings.put("PSPC", Arrays.asList("PSPC", "SPAC", "PUBLIC SERVICES AND PROCUREMENT CANADA", "SERVICES PUBLICS ET APPROVISIONNEMENT CANADA", "GOUVERNEMENT DU CANADA, SERVICES PUBLICS ET APPROVISIONNEMENT CANADA", "GOVERNMENT OF CANADA, PUBLIC SERVICES AND PROCUREMENT CANADA", "PSPC/SPAC")); - institutionMappings.put("RCMP", Arrays.asList("RCMP", "GRC", "ROYAL CANADIAN MOUNTED POLICE", "GENDARMERIE ROYALE DU CANADA", "RCMP/GRC")); - institutionMappings.put("SC", Arrays.asList("SC", "SC", "SERVICE CANADA", "SERVICE CANADA", "SC/SC")); - institutionMappings.put("SSC", Arrays.asList("SSC", "PSC", "SHARED SERVICES CANADA", "SERVICES PARTAGÉS CANADA", "SSC/PSC")); - institutionMappings.put("SSHRC", Arrays.asList("SSHRC", "CRSH", "SOCIAL SCIENCES AND HUMANITIES RESEARCH COUNCIL", "CONSEIL DE RECHERCHES EN SCIENCES HUMAINES", "SSHRC/CRSH")); - institutionMappings.put("SST", Arrays.asList("SST", "TSS", "SOCIAL SECURITY TRIBUNAL OF CANADA", "TRIBUNAL DE LA SÉCURITÉ SOCIALE DU CANADA", "SST/TSS")); - institutionMappings.put("STATCAN", Arrays.asList("STATCAN", "STATISTIQUE CANADA")); - institutionMappings.put("TBS", Arrays.asList("TBS", "SCT", "TREASURY BOARD OF CANADA SECRETARIAT", "SECRÉTARIAT DU CONSEIL DU TRÉSOR DU CANADA", "TBS/SCT")); - institutionMappings.put("TC", Arrays.asList("TC", "TC", "TRANSPORT CANADA", "TRANSPORTS CANADA")); - institutionMappings.put("VAC", Arrays.asList("VAC", "ACC", "VETERANS AFFAIRS CANADA", "ANCIENS COMBATTANTS CANADA", "VAC/ACC")); - institutionMappings.put("WAGE", Arrays.asList("WAGE", "FEGC", "WOMEN AND GENDER EQUALITY CANADA", "FEMMES ET ÉGALITÉ DES GENRES CANADA", "WAGE/FEGC")); - institutionMappings.put("WD", Arrays.asList("WD", "DEO", "WESTERN ECONOMIC DIVERSIFICATION CANADA", "DIVERSIFICATION DE L’ÉCONOMIE DE L’OUEST CANADA", "WD/DEO")); + private static final Logger LOG = LoggerFactory.getLogger(DashboardController.class); + + private final ProblemDateService problemDateService; + private final DashboardService dashboardService; + private final UserService userService; + private final ErrorKeywordService errorKeywordService; + private final MongoTemplate mongoTemplate; + + public DashboardController( + ProblemDateService problemDateService, + DashboardService dashboardService, + UserService userService, + ErrorKeywordService errorKeywordService, + MongoTemplate mongoTemplate) { + this.problemDateService = problemDateService; + this.dashboardService = dashboardService; + this.userService = userService; + this.errorKeywordService = errorKeywordService; + this.mongoTemplate = mongoTemplate; + } + + private static final Map> institutionMappings = new HashMap<>(); + private static final Map> sectionMappings = new HashMap<>(); + + static { + // Initialize section mappings + sectionMappings.put("disability", Arrays.asList("disability", "disability benefits")); + sectionMappings.put("news", Arrays.asList("news")); + + // Initialize institution mappings (kept as in the original file) + institutionMappings.put( + "AAFC", + Arrays.asList( + "AAFC", + "AAC", + "AGRICULTURE AND AGRI-FOOD CANADA", + "AGRICULTURE ET AGROALIMENTAIRE CANADA", + "AAFC/AAC")); + institutionMappings.put( + "ACOA", + Arrays.asList( + "ACOA", + "APECA", + "ATLANTIC CANADA OPPORTUNITIES AGENCY", + "AGENCE DE PROMOTION ÉCONOMIQUE DU CANADA ATLANTIQUE", + "ACOA/APECA")); + institutionMappings.put( + "ATSSC", + Arrays.asList( + "ATSSC", + "SCDATA", + "ADMINISTRATIVE TRIBUNALS SUPPORT SERVICE OF CANADA", + "SERVICE CANADIEN D’APPUI AUX TRIBUNAUX ADMINISTRATIFS", + "ATSSC/SCDATA")); + institutionMappings.put( + "CANNOR", + Arrays.asList( + "CANNOR", + "RNCAN", + "CANADIAN NORTHERN ECONOMIC DEVELOPMENT AGENCY", + "AGENCE CANADIENNE DE DÉVELOPPEMENT ÉCONOMIQUE DU NORD", + "CANNOR/RNCAN")); + institutionMappings.put( + "CATSA", + Arrays.asList( + "CATSA", + "ACSTA", + "CANADIAN AIR TRANSPORT SECURITY AUTHORITY", + "ADMINISTRATION CANADIENNE DE LA SÛRETÉ DU TRANSPORT AÉRIEN", + "CATSA/ACSTA")); + institutionMappings.put( + "CBSA", + Arrays.asList( + "CBSA", + "ASFC", + "CANADA BORDER SERVICES AGENCY", + "AGENCE DES SERVICES FRONTALIERS DU CANADA", + "CBSA/ASFC")); + institutionMappings.put( + "CCG", + Arrays.asList("CCG", "GCC", "CANADIAN COAST GUARD", "GARDE CÔTIÈRE CANADIENNE", "CCG/GCC")); + institutionMappings.put( + "CER", + Arrays.asList( + "CER", "REC", "CANADA ENERGY REGULATOR", "RÉGIE DE L'ÉNERGIE DU CANADA", "CER/REC")); + institutionMappings.put( + "CFIA", + Arrays.asList( + "CFIA", + "ACIA", + "CANADIAN FOOD INSPECTION AGENCY", + "AGENCE CANADIENNE D’INSPECTION DES ALIMENTS", + "CFIA/ACIA")); + institutionMappings.put( + "CIHR", + Arrays.asList( + "CIHR", + "IRSC", + "CANADIAN INSTITUTES OF HEALTH RESEARCH", + "INSTITUTS DE RECHERCHE EN SANTÉ DU CANADA", + "CIHR/IRSC")); + institutionMappings.put( + "CIPO", + Arrays.asList( + "CIPO", + "OPIC", + "CANADIAN INTELLECTUAL PROPERTY OFFICE", + "OFFICE DE LA PROPRIÉTÉ INTELLECTUELLE DU CANADA", + "CIPO/OPIC")); + institutionMappings.put( + "CIRNAC", + Arrays.asList( + "CIRNAC", + "RCAANC", + "CROWN-INDIGENOUS RELATIONS AND NORTHERN AFFAIRS CANADA", + "RELATIONS COURONNE-AUTOCHTONES ET AFFAIRES DU NORD CANADA", + "CIRNAC/RCAANC")); + institutionMappings.put( + "CRA", + Arrays.asList( + "CRA", "ARC", "CANADA REVENUE AGENCY", "AGENCE DU REVENU DU CANADA", "CRA/ARC")); + institutionMappings.put( + "CRTC", + Arrays.asList( + "CRTC", + "CRTC", + "CANADIAN RADIO-TELEVISION AND TELECOMMUNICATIONS COMMISSION", + "CONSEIL DE LA RADIODIFFUSION ET DES TÉLÉCOMMUNICATIONS CANADIENNES")); + institutionMappings.put( + "CSA", + Arrays.asList( + "CSA", "ASC", "CANADIAN SPACE AGENCY", "AGENCE SPATIALE CANADIENNE", "CSA/ASC")); + institutionMappings.put( + "CSC", + Arrays.asList( + "CSC", + "SCC", + "CORRECTIONAL SERVICE CANADA", + "SERVICE CORRECTIONNEL CANADA", + "CSC/SCC")); + institutionMappings.put( + "CSE", + Arrays.asList( + "CSE", + "CST", + "COMMUNICATIONS SECURITY ESTABLISHMENT", + "CENTRE DE LA SÉCURITÉ DES TÉLÉCOMMUNICATIONS", + "CSE/CST")); + institutionMappings.put( + "CSEC", + Arrays.asList( + "CSEC", + "CSTC", + "COMMUNICATIONS SECURITY ESTABLISHMENT CANADA", + "CENTRE DE LA SÉCURITÉ DES TÉLÉCOMMUNICATIONS CANADA", + "CSEC/CSTC")); + institutionMappings.put( + "CSPS", + Arrays.asList( + "CSPS", + "EFPC", + "CANADA SCHOOL OF PUBLIC SERVICE", + "ÉCOLE DE LA FONCTION PUBLIQUE DU CANADA", + "CSPS/EFPC")); + institutionMappings.put( + "DFO", + Arrays.asList( + "DFO", + "MPO", + "FISHERIES AND OCEANS CANADA", + "PÊCHES ET OCÉANS CANADA", + "DFO/MPO", + "GOVERNMENT OF CANADA, FISHERIES AND OCEANS CANADA, COMMUNICATIONS BRANCH")); + institutionMappings.put( + "DND", Arrays.asList("DND", "MDN", "NATIONAL DEFENCE", "DÉFENSE NATIONALE", "DND/MDN")); + institutionMappings.put( + "ECCC", + Arrays.asList( + "ECCC", + "ENVIRONMENT AND CLIMATE CHANGE CANADA", + "ENVIRONNEMENT ET CHANGEMENT CLIMATIQUE CANADA", + "ECCC")); + institutionMappings.put( + "ESDC", + Arrays.asList( + "ESDC", + "EDSC", + "EMPLOYMENT AND SOCIAL DEVELOPMENT CANADA", + "EMPLOI ET DÉVELOPPEMENT SOCIAL CANADA", + "ESDC/EDSC", + "EMPLOI ET DÉVÉLOPPEMENT SOCIALE CANADA")); + institutionMappings.put( + "FCAC", + Arrays.asList( + "FCAC", + "ACFC", + "FINANCIAL CONSUMER AGENCY OF CANADA", + "AGENCE DE LA CONSOMMATION EN MATIÈRE FINANCIÈRE DU CANADA", + "FCAC/ACFC")); + institutionMappings.put( + "FIN", + Arrays.asList( + "FIN", + "FIN", + "FINANCE CANADA", + "MINISTÈRE DES FINANCES CANADA", + "DEPARTMENT OF FINANCE CANADA", + "GOVERNMENT OF CANADA, DEPARTMENT OF FINANCE", + "MINISTÈRE DES FINANCES", + "FIN")); + institutionMappings.put( + "GAC", + Arrays.asList( + "GAC", "AMC", "GLOBAL AFFAIRS CANADA", "AFFAIRES MONDIALES CANADA", "GAC/AMC")); + institutionMappings.put( + "HC", Arrays.asList("HC", "SC", "HEALTH CANADA", "SANTÉ CANADA", "HC/SC")); + institutionMappings.put( + "HICC", + Arrays.asList( + "HICC", + "LICC", + "HOUSING, INFRASTRUCTURE AND COMMUNITIES CANADA", + "LOGEMENT, INFRASTRUCTURES ET COLLECTIVITÉS CANADA", + "HICC/LICC")); + institutionMappings.put( + "INFC", + Arrays.asList( + "INFC", "INFC", "INFRASTRUCTURE CANADA", "INFRASTRUCTURE CANADA", "INFC / INFC")); + institutionMappings.put( + "IOGC", + Arrays.asList( + "IOGC", + "BPGI", + "INDIAN OIL AND GAS CANADA", + "BUREAU DU PÉTROLE ET DU GAZ DES INDIENS", + "IOGC/BPGI")); + institutionMappings.put( + "IRCC", + Arrays.asList( + "IRCC", + "IRCC", + "IMMIGRATION, REFUGEES AND CITIZENSHIP CANADA", + "IMMIGRATION, RÉFUGIÉS ET CITOYENNETÉ CANADA")); + institutionMappings.put( + "ISC", + Arrays.asList( + "ISC", + "SAC", + "INDIGENOUS SERVICES CANADA", + "SERVICES AUX AUTOCHTONES CANADA", + "ISC/SAC")); + institutionMappings.put( + "ISED", + Arrays.asList( + "ISED", + "ISDE", + "INNOVATION, SCIENCE AND ECONOMIC DEVELOPMENT CANADA", + "INNOVATION, SCIENCES ET DÉVELOPPEMENT ÉCONOMIQUE CANADA", + "ISED/ISDE")); + institutionMappings.put( + "JUS", Arrays.asList("JUS", "JUSTICE CANADA", "MINISTÈRE DE LA JUSTICE CANADA", "JUS")); + institutionMappings.put( + "LAC", + Arrays.asList( + "LAC", + "BAC", + "LIBRARY AND ARCHIVES CANADA", + "BIBLIOTHÈQUE ET ARCHIVES CANADA", + "LAC/BAC")); + institutionMappings.put( + "NFB", + Arrays.asList("NFB", "ONF", "NATIONAL FILM BOARD", "OFFICE NATIONAL DU FILM", "NFB/ONF")); + institutionMappings.put( + "NRC", + Arrays.asList( + "NRC", + "CNRC", + "NATIONAL RESEARCH COUNCIL", + "CONSEIL NATIONAL DE RECHERCHES CANADA", + "NRC/CNRC")); + institutionMappings.put( + "NRCAN", + Arrays.asList( + "NRCAN", + "RNCAN", + "NATURAL RESOURCES CANADA", + "RESSOURCES NATURELLES CANADA", + "NRCAN/RNCAN")); + institutionMappings.put( + "NSERC", + Arrays.asList( + "NSERC", + "CRSNG", + "NATURAL SCIENCES AND ENGINEERING RESEARCH CANADA", + "CONSEIL DE RECHERCHES EN SCIENCES NATURELLES ET EN GÉNIE DU CANADA", + "NSERC/CRSNG")); + institutionMappings.put( + "OMBDNDCAF", + Arrays.asList( + "OMBDNDCAF", + "OMBMDNFAC", + "DND/CAF OMBUDSMAN", + "OMBUDSMAN DU MDN/FAC", + "OFFICE OF THE NATIONAL DEFENCE AND CANADIAN ARMED FORCES OMBUDSMAN", + "BUREAU DE L'OMBUDSMAN DE LA DÉFENSE NATIONALE ET DES FORCES ARMÉES CANADIENNES", + "OMBDNDCAF/OMBMDNFAC")); + institutionMappings.put( + "OSB", + Arrays.asList( + "OSB", + "BSF", + "SUPERINTENDENT OF BANKRUPTCY CANADA", + "BUREAU DU SURINTENDANT DES FAILLITES CANADA", + "OSB/BSF")); + institutionMappings.put( + "PBC", + Arrays.asList( + "PBC", + "CLCC", + "PAROLE BOARD OF CANADA", + "COMMISSION DES LIBÉRATIONS CONDITIONNELLES DU CANADA", + "PBC/CLCC")); + institutionMappings.put("PC", Arrays.asList("PC", "PC", "PARCS CANADA", "PARKS CANADA")); + institutionMappings.put( + "PCH", Arrays.asList("PCH", "PCH", "CANADIAN HERITAGE", "PATRIMOINE CANADIEN")); + institutionMappings.put( + "PCO", + Arrays.asList("PCO", "BCP", "PRIVY COUNCIL OFFICE", "BUREAU DU CONSEIL PRIVÉ", "PCO/BCP")); + institutionMappings.put( + "PHAC", + Arrays.asList( + "PHAC", + "ASPC", + "PUBLIC HEALTH AGENCY OF CANADA", + "AGENCE DE LA SÉAUTÉ PUBLIQUE DU CANADA", + "PHAC/ASPC")); + institutionMappings.put( + "PS", + Arrays.asList("PS", "SP", "PUBLIC SAFETY CANADA", "SÉCURITÉ PUBLIQUE CANADA", "PS/SP")); + institutionMappings.put( + "PSC", + Arrays.asList( + "PSC", + "CFP", + "PUBLIC SERVICE COMMISSION OF CANADA", + "COMMISSION DE LA FONCTION PUBLIQUE DU CANADA", + "PSC/CFP")); + institutionMappings.put( + "PSPC", + Arrays.asList( + "PSPC", + "SPAC", + "PUBLIC SERVICES AND PROCUREMENT CANADA", + "SERVICES PUBLICS ET APPROVISIONNEMENT CANADA", + "GOUVERNEMENT DU CANADA, SERVICES PUBLICS ET APPROVISIONNEMENT CANADA", + "GOVERNMENT OF CANADA, PUBLIC SERVICES AND PROCUREMENT CANADA", + "PSPC/SPAC")); + institutionMappings.put( + "RCMP", + Arrays.asList( + "RCMP", + "GRC", + "ROYAL CANADIAN MOUNTED POLICE", + "GENDARMERIE ROYALE DU CANADA", + "RCMP/GRC")); + institutionMappings.put( + "SC", Arrays.asList("SC", "SC", "SERVICE CANADA", "SERVICE CANADA", "SC/SC")); + institutionMappings.put( + "SSC", + Arrays.asList( + "SSC", "PSC", "SHARED SERVICES CANADA", "SERVICES PARTAGÉS CANADA", "SSC/PSC")); + institutionMappings.put( + "SSHRC", + Arrays.asList( + "SSHRC", + "CRSH", + "SOCIAL SCIENCES AND HUMANITIES RESEARCH COUNCIL", + "CONSEIL DE RECHERCHES EN SCIENCES HUMAINES", + "SSHRC/CRSH")); + institutionMappings.put( + "SST", + Arrays.asList( + "SST", + "TSS", + "SOCIAL SECURITY TRIBUNAL OF CANADA", + "TRIBUNAL DE LA SÉCURITÉ SOCIALE DU CANADA", + "SST/TSS")); + institutionMappings.put("STATCAN", Arrays.asList("STATCAN", "STATISTIQUE CANADA")); + institutionMappings.put( + "TBS", + Arrays.asList( + "TBS", + "SCT", + "TREASURY BOARD OF CANADA SECRETARIAT", + "SECRÉTARIAT DU CONSEIL DU TRÉSOR DU CANADA", + "TBS/SCT")); + institutionMappings.put( + "TC", Arrays.asList("TC", "TC", "TRANSPORT CANADA", "TRANSPORTS CANADA")); + institutionMappings.put( + "VAC", + Arrays.asList( + "VAC", "ACC", "VETERANS AFFAIRS CANADA", "ANCIENS COMBATTANTS CANADA", "VAC/ACC")); + institutionMappings.put( + "WAGE", + Arrays.asList( + "WAGE", + "FEGC", + "WOMEN AND GENDER EQUALITY CANADA", + "FEMMES ET ÉGALITÉ DES GENRES CANADA", + "WAGE/FEGC")); + institutionMappings.put( + "WD", + Arrays.asList( + "WD", + "DEO", + "WESTERN ECONOMIC DIVERSIFICATION CANADA", + "DIVERSIFICATION DE L’ÉCONOMIE DE L’OUEST CANADA", + "WD/DEO")); + } + + @RequestMapping(value = "/pageFeedback/totalCommentsCount") + @ResponseBody + public String totalCommentsCount(HttpServletRequest request) { + String comments = request.getParameter("comments"); + String startDate = request.getParameter("startDate"); + String endDate = request.getParameter("endDate"); + String theme = request.getParameter("theme"); + String section = request.getParameter("section"); + String language = request.getParameter("language"); + String url = request.getParameter("url"); + String department = request.getParameter("department"); + boolean error_keyword = "true".equals(request.getParameter("error_keyword")); + + Totals t = + getTotalPagesAndComments( + comments, startDate, endDate, theme, section, language, url, department, error_keyword); + return String.valueOf(t.comments()); + } + + @RequestMapping(value = "/pageFeedback/totalPagesCount") + @ResponseBody + public String totalPagesCount(HttpServletRequest request) { + String comments = request.getParameter("comments"); + String startDate = request.getParameter("startDate"); + String endDate = request.getParameter("endDate"); + String theme = request.getParameter("theme"); + String section = request.getParameter("section"); + String language = request.getParameter("language"); + String url = request.getParameter("url"); + String department = request.getParameter("department"); + boolean error_keyword = "true".equals(request.getParameter("error_keyword")); + + Totals t = + getTotalPagesAndComments( + comments, startDate, endDate, theme, section, language, url, department, error_keyword); + return String.valueOf(t.pages()); + } + + @GetMapping(value = "/dashboard") + public ModelAndView pageFeedback(HttpServletRequest request) { + var mav = new ModelAndView(); + String lang = (String) request.getSession().getAttribute("lang"); + mav.addObject("lang", lang); + var dateMap = problemDateService.getProblemDates(); + if (dateMap != null) { + mav.addObject("earliestDate", dateMap.get("earliestDate")); + var latestDate = LocalDate.parse(dateMap.get("latestDate"), DateTimeFormatter.ISO_LOCAL_DATE); + var previousDate = latestDate.minusDays(1); + var modifiedLatestDate = previousDate.format(DateTimeFormatter.ISO_LOCAL_DATE); + mav.addObject("latestDate", modifiedLatestDate); + } else { + mav.addObject("earliestDate", "N/A"); + mav.addObject("latestDate", "N/A"); } - - @RequestMapping(value = "/pageFeedback/totalCommentsCount") - @ResponseBody - public String totalCommentsCount(HttpServletRequest request) { - String comments = request.getParameter("comments"); - String startDate = request.getParameter("startDate"); - String endDate = request.getParameter("endDate"); - String theme = request.getParameter("theme"); - String section = request.getParameter("section"); - String language = request.getParameter("language"); - String url = request.getParameter("url"); - String department = request.getParameter("department"); - boolean error_keyword = "true".equals(request.getParameter("error_keyword")); - - Totals t = getTotalPagesAndComments(comments, startDate, endDate, theme, section, language, url, department, error_keyword); - return String.valueOf(t.comments()); + mav.setViewName("pageFeedbackDashboard_" + lang); + return mav; + } + + @GetMapping(value = "/chartData") + @ResponseBody + public List> commentsByDate(HttpServletRequest request) { + String error_keyword_param = request.getParameter("error_keyword"); + boolean error_keyword = "true".equals(error_keyword_param); + String comments = request.getParameter("comments"); + String startDate = request.getParameter("startDate"); + String endDate = request.getParameter("endDate"); + String theme = request.getParameter("theme"); + String section = request.getParameter("section"); + String language = request.getParameter("language"); + String url = request.getParameter("url"); + String department = request.getParameter("department"); + + boolean useDatabase = + error_keyword + || (comments != null + && !comments.trim().isEmpty() + && !"null".equalsIgnoreCase(comments.trim())); + + if (useDatabase) { + var criteria = + buildFilterCriteria(startDate, endDate, theme, section, language, url, department); + var finalCriteria = applyRegexCriteria(criteria, comments, error_keyword); + + var groupByDate = Aggregation.group("problemDate").count().as("comments"); + var sortByDate = Aggregation.sort(Sort.Direction.ASC, "_id"); + var aggResults = + mongoTemplate.aggregate( + Aggregation.newAggregation(Aggregation.match(finalCriteria), groupByDate, sortByDate), + "problem", + Document.class); + + var dailyCommentsList = new ArrayList>(); + for (Document doc : aggResults) { + var map = new HashMap(); + map.put("date", doc.getString("_id")); + map.put("comments", doc.getInteger("comments", 0)); + dailyCommentsList.add(map); + } + return dailyCommentsList; } - @RequestMapping(value = "/pageFeedback/totalPagesCount") - @ResponseBody - public String totalPagesCount(HttpServletRequest request) { - String comments = request.getParameter("comments"); - String startDate = request.getParameter("startDate"); - String endDate = request.getParameter("endDate"); - String theme = request.getParameter("theme"); - String section = request.getParameter("section"); - String language = request.getParameter("language"); - String url = request.getParameter("url"); - String department = request.getParameter("department"); - boolean error_keyword = "true".equals(request.getParameter("error_keyword")); - - Totals t = getTotalPagesAndComments(comments, startDate, endDate, theme, section, language, url, department, error_keyword); - return String.valueOf(t.pages()); - } - - - @GetMapping(value = "/dashboard") - public ModelAndView pageFeedback(HttpServletRequest request) { - var mav = new ModelAndView(); - String lang = (String) request.getSession().getAttribute("lang"); - mav.addObject("lang", lang); - var dateMap = problemDateService.getProblemDates(); - if (dateMap != null) { - mav.addObject("earliestDate", dateMap.get("earliestDate")); - var latestDate = LocalDate.parse(dateMap.get("latestDate"), DateTimeFormatter.ISO_LOCAL_DATE); - var previousDate = latestDate.minusDays(1); - var modifiedLatestDate = previousDate.format(DateTimeFormatter.ISO_LOCAL_DATE); - mav.addObject("latestDate", modifiedLatestDate); - } else { - mav.addObject("earliestDate", "N/A"); - mav.addObject("latestDate", "N/A"); - } - mav.setViewName("pageFeedbackDashboard_" + lang); - return mav; + var stats = dashboardService.getDashboardStats(); + var problemsByDate = + applyFilters( + new ArrayList<>(stats.problemsByDate()), + department, + startDate, + endDate, + language, + url, + section, + theme); + + var dateToCommentCountMap = new HashMap(); + for (Problem problem : problemsByDate) { + if (problem != null && problem.getProblemDate() != null) { + dateToCommentCountMap.merge( + problem.getProblemDate(), problem.getUrlEntries(), Integer::sum); + } } - @GetMapping(value = "/chartData") - @ResponseBody - public List> commentsByDate(HttpServletRequest request) { - String error_keyword_param = request.getParameter("error_keyword"); - boolean error_keyword = "true".equals(error_keyword_param); - String comments = request.getParameter("comments"); - String startDate = request.getParameter("startDate"); - String endDate = request.getParameter("endDate"); - String theme = request.getParameter("theme"); - String section = request.getParameter("section"); - String language = request.getParameter("language"); - String url = request.getParameter("url"); - String department = request.getParameter("department"); - - boolean useDatabase = error_keyword || (comments != null && !comments.trim().isEmpty() && !"null".equalsIgnoreCase(comments.trim())); - - if (useDatabase) { - var criteria = buildFilterCriteria(startDate, endDate, theme, section, language, url, department); - var finalCriteria = applyRegexCriteria(criteria, comments, error_keyword); - - var groupByDate = Aggregation.group("problemDate").count().as("comments"); - var sortByDate = Aggregation.sort(Sort.Direction.ASC, "_id"); - var aggResults = mongoTemplate.aggregate( - Aggregation.newAggregation(Aggregation.match(finalCriteria), groupByDate, sortByDate), - "problem", Document.class); - - var dailyCommentsList = new ArrayList>(); - for (Document doc : aggResults) { - var map = new HashMap(); - map.put("date", doc.getString("_id")); - map.put("comments", doc.getInteger("comments", 0)); - dailyCommentsList.add(map); - } - return dailyCommentsList; - } - - var stats = dashboardService.getDashboardStats(); - var problemsByDate = applyFilters(new ArrayList<>(stats.problemsByDate()), department, startDate, endDate, language, url, section, theme); - - var dateToCommentCountMap = new HashMap(); - for (Problem problem : problemsByDate) { - if (problem != null && problem.getProblemDate() != null) { - dateToCommentCountMap.merge(problem.getProblemDate(), problem.getUrlEntries(), Integer::sum); - } - } - - var dailyCommentsList = new ArrayList>(); - dateToCommentCountMap.forEach((date, count) -> { - var entry = new HashMap(); - entry.put("date", date); - entry.put("comments", count); - dailyCommentsList.add(entry); + var dailyCommentsList = new ArrayList>(); + dateToCommentCountMap.forEach( + (date, count) -> { + var entry = new HashMap(); + entry.put("date", date); + entry.put("comments", count); + dailyCommentsList.add(entry); }); - dailyCommentsList.sort(Comparator.comparing(map -> (String) map.get("date"))); - return dailyCommentsList; + dailyCommentsList.sort(Comparator.comparing(map -> (String) map.get("date"))); + return dailyCommentsList; + } + + @GetMapping(value = "/dashboardData") + @ResponseBody + public DataTablesOutput getDashboardData( + @Valid DataTablesInput input, HttpServletRequest request) { + String pageLang = (String) request.getSession().getAttribute("lang"); + String department = request.getParameter("department"); + String startDate = request.getParameter("startDate"); + String endDate = request.getParameter("endDate"); + String language = request.getParameter("language"); + String url = request.getParameter("url"); + String comments = request.getParameter("comments"); + String section = request.getParameter("section"); + String theme = request.getParameter("theme"); + boolean error_keyword = "true".equals(request.getParameter("error_keyword")); + + boolean hasRegexFilter = + error_keyword + || (comments != null + && !comments.trim().isEmpty() + && !"null".equalsIgnoreCase(comments.trim())); + + if (hasRegexFilter) { + return getDashboardDataViaAggregation( + input, + pageLang, + startDate, + endDate, + theme, + section, + language, + url, + department, + comments, + error_keyword); } - @GetMapping(value = "/dashboardData") - @ResponseBody - public DataTablesOutput getDashboardData(@Valid DataTablesInput input, HttpServletRequest request) { - String pageLang = (String) request.getSession().getAttribute("lang"); - String department = request.getParameter("department"); - String startDate = request.getParameter("startDate"); - String endDate = request.getParameter("endDate"); - String language = request.getParameter("language"); - String url = request.getParameter("url"); - String comments = request.getParameter("comments"); - String section = request.getParameter("section"); - String theme = request.getParameter("theme"); - boolean error_keyword = "true".equals(request.getParameter("error_keyword")); - - boolean hasRegexFilter = error_keyword || (comments != null && !comments.trim().isEmpty() && !"null".equalsIgnoreCase(comments.trim())); - - if (hasRegexFilter) { - return getDashboardDataViaAggregation(input, pageLang, startDate, endDate, theme, section, language, url, department, comments, error_keyword); - } - - var stats = dashboardService.getDashboardStats(); - var filtered = applyFilters(new ArrayList<>(stats.problemsByDate()), department, startDate, endDate, language, url, section, theme); - var merged = mergeProblems(filtered); - merged.sort(Comparator.comparingInt(Problem::getUrlEntries).reversed()); - - int filteredTotalPages = merged.size(); - var page = merged.stream().skip(input.getStart()).limit(input.getLength()).collect(Collectors.toList()); - - var output = new DataTablesOutput(); - output.setData(page); - output.setDraw(input.getDraw()); - output.setRecordsTotal(filteredTotalPages); - output.setRecordsFiltered(filteredTotalPages); - setInstitutionNames(output, pageLang); - return output; + var stats = dashboardService.getDashboardStats(); + var filtered = + applyFilters( + new ArrayList<>(stats.problemsByDate()), + department, + startDate, + endDate, + language, + url, + section, + theme); + var merged = mergeProblems(filtered); + merged.sort(Comparator.comparingInt(Problem::getUrlEntries).reversed()); + + int filteredTotalPages = merged.size(); + var page = + merged.stream() + .skip(input.getStart()) + .limit(input.getLength()) + .collect(Collectors.toList()); + + var output = new DataTablesOutput(); + output.setData(page); + output.setDraw(input.getDraw()); + output.setRecordsTotal(filteredTotalPages); + output.setRecordsFiltered(filteredTotalPages); + setInstitutionNames(output, pageLang); + return output; + } + + private DataTablesOutput getDashboardDataViaAggregation( + DataTablesInput input, + String pageLang, + String startDate, + String endDate, + String theme, + String section, + String language, + String url, + String department, + String comments, + boolean error_keyword) { + + var criteria = + buildFilterCriteria(startDate, endDate, theme, section, language, url, department); + criteria = applyRegexCriteria(criteria, comments, error_keyword); + + var match = Aggregation.match(criteria); + var groupByUrl = + Aggregation.group("url") + .first("url") + .as("url") + .first("problemDate") + .as("problemDate") + .first("institution") + .as("institution") + .first("title") + .as("title") + .first("language") + .as("language") + .first("section") + .as("section") + .first("theme") + .as("theme") + .count() + .as("urlEntries"); + var sortDesc = Aggregation.sort(Sort.Direction.DESC, "urlEntries"); + + var page = + mongoTemplate + .aggregate( + Aggregation.newAggregation( + match, + groupByUrl, + sortDesc, + Aggregation.skip((long) input.getStart()), + Aggregation.limit(input.getLength())), + "problem", + Problem.class) + .getMappedResults(); + + var totalsDoc = + mongoTemplate + .aggregate( + Aggregation.newAggregation( + match, + groupByUrl, + Aggregation.group().count().as("pages").sum("urlEntries").as("comments")), + "problem", + Document.class) + .getUniqueMappedResult(); + + int totalP = 0; + if (totalsDoc != null) { + totalP = totalsDoc.getInteger("pages", 0); } - private DataTablesOutput getDashboardDataViaAggregation( - DataTablesInput input, String pageLang, - String startDate, String endDate, String theme, String section, - String language, String url, String department, - String comments, boolean error_keyword) { - - var criteria = buildFilterCriteria(startDate, endDate, theme, section, language, url, department); - criteria = applyRegexCriteria(criteria, comments, error_keyword); - - var match = Aggregation.match(criteria); - var groupByUrl = Aggregation.group("url") - .first("url").as("url") - .first("problemDate").as("problemDate") - .first("institution").as("institution") - .first("title").as("title") - .first("language").as("language") - .first("section").as("section") - .first("theme").as("theme") - .count().as("urlEntries"); - var sortDesc = Aggregation.sort(Sort.Direction.DESC, "urlEntries"); - - var page = mongoTemplate.aggregate( - Aggregation.newAggregation(match, groupByUrl, sortDesc, Aggregation.skip((long) input.getStart()), Aggregation.limit(input.getLength())), - "problem", Problem.class).getMappedResults(); - - var totalsDoc = mongoTemplate.aggregate( - Aggregation.newAggregation(match, groupByUrl, Aggregation.group().count().as("pages").sum("urlEntries").as("comments")), - "problem", Document.class).getUniqueMappedResult(); - - int totalP = 0; - if (totalsDoc != null) { - totalP = totalsDoc.getInteger("pages", 0); - } - - var output = new DataTablesOutput(); - output.setData(page); - output.setDraw(input.getDraw()); - output.setRecordsTotal(totalP); - output.setRecordsFiltered(totalP); - setInstitutionNames(output, pageLang); - return output; + var output = new DataTablesOutput(); + output.setData(page); + output.setDraw(input.getDraw()); + output.setRecordsTotal(totalP); + output.setRecordsFiltered(totalP); + setInstitutionNames(output, pageLang); + return output; + } + + private Criteria buildFilterCriteria( + String startDate, + String endDate, + String theme, + String section, + String language, + String url, + String department) { + var criteria = Criteria.where("processed").is("true"); + var formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + if (startDate != null && !startDate.isEmpty() && endDate != null && !endDate.isEmpty()) { + criteria.and("problemDate").gte(startDate).lte(endDate); } - - private Criteria buildFilterCriteria(String startDate, String endDate, String theme, String section, String language, String url, String department) { - var criteria = Criteria.where("processed").is("true"); - var formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - if (startDate != null && !startDate.isEmpty() && endDate != null && !endDate.isEmpty()) { - criteria.and("problemDate").gte(startDate).lte(endDate); - } - if (theme != null && !theme.isEmpty()) criteria.and("theme").is(theme); - if (section != null && !section.isEmpty()) { - criteria.and("section").in(sectionMappings.getOrDefault(section.toLowerCase(), Collections.singletonList(section))); - } - if (language != null && !language.isEmpty()) criteria.and("language").is(language); - if (url != null && !url.isEmpty()) criteria.and("url").regex(url, "i"); - if (department != null && !department.isEmpty()) { - var variations = new HashSet(); - for (List list : institutionMappings.values()) { - if (list.stream().anyMatch(v -> v.equalsIgnoreCase(department))) variations.addAll(list); - } - if (!variations.isEmpty()) criteria.and("institution").in(variations); - } - return criteria; + if (theme != null && !theme.isEmpty()) criteria.and("theme").is(theme); + if (section != null && !section.isEmpty()) { + criteria + .and("section") + .in( + sectionMappings.getOrDefault( + section.toLowerCase(), Collections.singletonList(section))); } - - private List applyFilters(List problems, String department, String startDate, String endDate, String language, String url, String section, String theme) { - var formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - var stream = problems.stream(); - - if (department != null && !department.isEmpty()) { - var variations = new HashSet(); - for (List list : institutionMappings.values()) { - if (list.stream().anyMatch(v -> v.equalsIgnoreCase(department))) variations.addAll(list); - } - if (!variations.isEmpty()) stream = stream.filter(p -> variations.contains(p.getInstitution())); - } - - if (startDate != null && endDate != null) { - var start = LocalDate.parse(startDate, formatter); - var end = LocalDate.parse(endDate, formatter); - stream = stream.filter(p -> { - try { - LocalDate d = LocalDate.parse(p.getProblemDate(), formatter); - return !d.isBefore(start) && !d.isAfter(end); - } catch (Exception e) { return false; } - }); - } - if (language != null && !language.isEmpty()) stream = stream.filter(p -> language.equals(p.getLanguage())); - if (url != null && !url.isEmpty()) stream = stream.filter(p -> p.getUrl().toLowerCase().contains(url.toLowerCase())); - if (section != null && !section.isEmpty()) { - List sections = sectionMappings.getOrDefault(section.toLowerCase(), Collections.singletonList(section)); - stream = stream.filter(p -> sections.contains(p.getSection())); - } - if (theme != null && !theme.isEmpty()) stream = stream.filter(p -> theme.equals(p.getTheme())); - - return stream.collect(Collectors.toList()); + if (language != null && !language.isEmpty()) criteria.and("language").is(language); + if (url != null && !url.isEmpty()) criteria.and("url").regex(url, "i"); + if (department != null && !department.isEmpty()) { + var variations = new HashSet(); + for (List list : institutionMappings.values()) { + if (list.stream().anyMatch(v -> v.equalsIgnoreCase(department))) variations.addAll(list); + } + if (!variations.isEmpty()) criteria.and("institution").in(variations); } - - private List mergeProblems(List problems) { - var map = new LinkedHashMap(); - for (Problem p : problems) { - map.merge(p.getUrl(), p, (o, n) -> { - Problem updated = new Problem(o); - updated.setUrlEntries(o.getUrlEntries() + n.getUrlEntries()); - return updated; - }); - } - return new ArrayList<>(map.values()); + return criteria; + } + + private List applyFilters( + List problems, + String department, + String startDate, + String endDate, + String language, + String url, + String section, + String theme) { + var formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + var stream = problems.stream(); + + if (department != null && !department.isEmpty()) { + var variations = new HashSet(); + for (List list : institutionMappings.values()) { + if (list.stream().anyMatch(v -> v.equalsIgnoreCase(department))) variations.addAll(list); + } + if (!variations.isEmpty()) + stream = stream.filter(p -> variations.contains(p.getInstitution())); } - private void setInstitutionNames(DataTablesOutput output, String lang) { - for (Problem p : output.getData()) { - for (List variations : institutionMappings.values()) { - if (variations.contains(p.getInstitution())) { - p.setInstitution(variations.get("fr".equalsIgnoreCase(lang) ? 1 : 0)); - break; + if (startDate != null && endDate != null) { + var start = LocalDate.parse(startDate, formatter); + var end = LocalDate.parse(endDate, formatter); + stream = + stream.filter( + p -> { + try { + LocalDate d = LocalDate.parse(p.getProblemDate(), formatter); + return !d.isBefore(start) && !d.isAfter(end); + } catch (Exception e) { + return false; } - } - } + }); } - - private String escapeSpecialRegexCharacters(String input) { - return input.replaceAll("([\\\\.^$|()\\[\\]{}*+?])", "\\\\$1"); + if (language != null && !language.isEmpty()) + stream = stream.filter(p -> language.equals(p.getLanguage())); + if (url != null && !url.isEmpty()) + stream = stream.filter(p -> p.getUrl().toLowerCase().contains(url.toLowerCase())); + if (section != null && !section.isEmpty()) { + List sections = + sectionMappings.getOrDefault(section.toLowerCase(), Collections.singletonList(section)); + stream = stream.filter(p -> sections.contains(p.getSection())); } - - private Criteria applyRegexCriteria(Criteria criteria, String comments, boolean error_keyword) { - var regexCriteria = new ArrayList(); - if (error_keyword) { - var keywords = new HashSet(); - keywords.addAll(errorKeywordService.getEnglishKeywords()); - keywords.addAll(errorKeywordService.getFrenchKeywords()); - keywords.addAll(errorKeywordService.getBilingualKeywords()); - if (!keywords.isEmpty()) { - String combinedRegex = keywords.stream().map(Pattern::quote).collect(Collectors.joining("|")); - regexCriteria.add(Criteria.where("problemDetails").regex(combinedRegex, "i")); - } - } - if (comments != null && !comments.trim().isEmpty() && !"null".equalsIgnoreCase(comments.trim())) { - regexCriteria.add(Criteria.where("problemDetails").regex(escapeSpecialRegexCharacters(comments.trim()), "i")); - } - if (!regexCriteria.isEmpty()) { - criteria = new Criteria().andOperator(criteria, new Criteria().andOperator(regexCriteria.toArray(new Criteria[0]))); + if (theme != null && !theme.isEmpty()) stream = stream.filter(p -> theme.equals(p.getTheme())); + + return stream.collect(Collectors.toList()); + } + + private List mergeProblems(List problems) { + var map = new LinkedHashMap(); + for (Problem p : problems) { + map.merge( + p.getUrl(), + p, + (o, n) -> { + Problem updated = new Problem(o); + updated.setUrlEntries(o.getUrlEntries() + n.getUrlEntries()); + return updated; + }); + } + return new ArrayList<>(map.values()); + } + + private void setInstitutionNames(DataTablesOutput output, String lang) { + for (Problem p : output.getData()) { + for (List variations : institutionMappings.values()) { + if (variations.contains(p.getInstitution())) { + p.setInstitution(variations.get("fr".equalsIgnoreCase(lang) ? 1 : 0)); + break; } - return criteria; + } } - - //helper to record totals for pages and comments - private record Totals(int pages, int comments) { + } + + private String escapeSpecialRegexCharacters(String input) { + return input.replaceAll("([\\\\.^$|()\\[\\]{}*+?])", "\\\\$1"); + } + + private Criteria applyRegexCriteria(Criteria criteria, String comments, boolean error_keyword) { + var regexCriteria = new ArrayList(); + if (error_keyword) { + var keywords = new HashSet(); + keywords.addAll(errorKeywordService.getEnglishKeywords()); + keywords.addAll(errorKeywordService.getFrenchKeywords()); + keywords.addAll(errorKeywordService.getBilingualKeywords()); + if (!keywords.isEmpty()) { + String combinedRegex = + keywords.stream().map(Pattern::quote).collect(Collectors.joining("|")); + regexCriteria.add(Criteria.where("problemDetails").regex(combinedRegex, "i")); + } } - - //Calculate total pages and comments based on filters, optimizing for cases with regex filters by using MongoDB aggregation, and in-memory filtering/merging when no regex filters are applied - private Totals getTotalPagesAndComments(String comments, String startDate, String endDate, String theme, String section, String language, String urlParam, String department, boolean error_keyword) { - boolean hasRegexFilter = error_keyword || (comments != null && !comments.trim().isEmpty() && !"null".equalsIgnoreCase(comments.trim())); - - if (hasRegexFilter) { - var criteria = buildFilterCriteria(startDate, endDate, theme, section, language, urlParam, department); - criteria = applyRegexCriteria(criteria, comments, error_keyword); - - var match = Aggregation.match(criteria); - var groupByUrl = Aggregation.group("url").count().as("urlEntries"); - - var totalsDoc = mongoTemplate.aggregate( - Aggregation.newAggregation(match, groupByUrl, Aggregation.group().count().as("pages").sum("urlEntries").as("comments")), - "problem", Document.class).getUniqueMappedResult(); - - int pages = totalsDoc != null ? totalsDoc.getInteger("pages", 0) : 0; - int commentsCount = totalsDoc != null ? totalsDoc.getInteger("comments", 0) : 0; - return new Totals(pages, commentsCount); - } - // If no regex filter, we can use in-memory filtering and merging - var stats = dashboardService.getDashboardStats(); - var filtered = applyFilters(new ArrayList<>(stats.problemsByDate()), department, startDate, endDate, language, urlParam, section, theme); - var merged = mergeProblems(filtered); - int pages = merged.size(); - int commentsCount = merged.stream().mapToInt(Problem::getUrlEntries).sum(); - return new Totals(pages, commentsCount); + if (comments != null + && !comments.trim().isEmpty() + && !"null".equalsIgnoreCase(comments.trim())) { + regexCriteria.add( + Criteria.where("problemDetails") + .regex(escapeSpecialRegexCharacters(comments.trim()), "i")); } - - @GetMapping("/dashboard/exportExcel") - public void exportExcel(HttpServletRequest request, HttpServletResponse response) - throws IOException { - String pageLang = (String) request.getSession().getAttribute("lang"); - String filename = buildExportFilename(pageLang, ".xlsx"); - response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); - response.setHeader("Content-Disposition", "attachment; filename=\"" + filename + "\""); - - var results = getAggregatedExportData(request); - - try (SXSSFWorkbook workbook = new SXSSFWorkbook(100); - ServletOutputStream outputStream = response.getOutputStream()) { - - Sheet sheet = workbook.createSheet("Dashboard Data"); - - String[] columns = {"Department", "URL", "Total Comments", "Language", "Section", "Theme"}; - var headerRow = sheet.createRow(0); - for (int i = 0; i < columns.length; i++) { - headerRow.createCell(i).setCellValue(columns[i]); - } - - int rowNum = 1; - for (Problem p : results) { - Row row = sheet.createRow(rowNum++); - row.createCell(0).setCellValue(resolveInstitutionName(p.getInstitution(), pageLang)); - row.createCell(1).setCellValue(p.getUrl()); - row.createCell(2).setCellValue(p.getUrlEntries()); - row.createCell(3).setCellValue(p.getLanguage()); - row.createCell(4).setCellValue(p.getSection()); - row.createCell(5).setCellValue(p.getTheme()); - - if (rowNum % 100 == 0) { - try { - ((SXSSFSheet) sheet).flushRows(100); - } catch (IOException e) { - LOG.error("Error flushing rows", e); - } - } - } - - workbook.write(outputStream); - } catch (Exception e) { - LOG.error("Error exporting Excel", e); - response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - } + if (!regexCriteria.isEmpty()) { + criteria = + new Criteria() + .andOperator( + criteria, new Criteria().andOperator(regexCriteria.toArray(new Criteria[0]))); } - - @GetMapping("/dashboard/exportCSV") - public void exportCSV(HttpServletRequest request, HttpServletResponse response) - throws IOException { - String pageLang = (String) request.getSession().getAttribute("lang"); - String filename = buildExportFilename(pageLang, ".csv"); - response.setCharacterEncoding("UTF-8"); - response.setContentType("text/csv; charset=UTF-8"); - response.setHeader("Content-Disposition", "attachment; filename*=UTF-8''" + filename); - - var results = getAggregatedExportData(request); - - try (Writer writer = response.getWriter()) { - writer.write("\uFEFF"); - writer.write("Department,URL,Total Comments,Language,Section,Theme\n"); - - for (Problem p : results) { - writer.write(String.format("%s,%s,%d,%s,%s,%s\n", - escapeCSV(resolveInstitutionName(p.getInstitution(), pageLang)), - escapeCSV(p.getUrl()), - p.getUrlEntries(), - escapeCSV(p.getLanguage()), - escapeCSV(p.getSection()), - escapeCSV(p.getTheme()))); - } - } + return criteria; + } + + // helper to record totals for pages and comments + private record Totals(int pages, int comments) {} + + // Calculate total pages and comments based on filters, optimizing for cases with regex filters by + // using MongoDB aggregation, and in-memory filtering/merging when no regex filters are applied + private Totals getTotalPagesAndComments( + String comments, + String startDate, + String endDate, + String theme, + String section, + String language, + String urlParam, + String department, + boolean error_keyword) { + boolean hasRegexFilter = + error_keyword + || (comments != null + && !comments.trim().isEmpty() + && !"null".equalsIgnoreCase(comments.trim())); + + if (hasRegexFilter) { + var criteria = + buildFilterCriteria(startDate, endDate, theme, section, language, urlParam, department); + criteria = applyRegexCriteria(criteria, comments, error_keyword); + + var match = Aggregation.match(criteria); + var groupByUrl = Aggregation.group("url").count().as("urlEntries"); + + var totalsDoc = + mongoTemplate + .aggregate( + Aggregation.newAggregation( + match, + groupByUrl, + Aggregation.group().count().as("pages").sum("urlEntries").as("comments")), + "problem", + Document.class) + .getUniqueMappedResult(); + + int pages = totalsDoc != null ? totalsDoc.getInteger("pages", 0) : 0; + int commentsCount = totalsDoc != null ? totalsDoc.getInteger("comments", 0) : 0; + return new Totals(pages, commentsCount); } + // If no regex filter, we can use in-memory filtering and merging + var stats = dashboardService.getDashboardStats(); + var filtered = + applyFilters( + new ArrayList<>(stats.problemsByDate()), + department, + startDate, + endDate, + language, + urlParam, + section, + theme); + var merged = mergeProblems(filtered); + int pages = merged.size(); + int commentsCount = merged.stream().mapToInt(Problem::getUrlEntries).sum(); + return new Totals(pages, commentsCount); + } + + @GetMapping("/dashboard/exportExcel") + public void exportExcel(HttpServletRequest request, HttpServletResponse response) + throws IOException { + String pageLang = (String) request.getSession().getAttribute("lang"); + String filename = buildExportFilename(pageLang, ".xlsx"); + response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); + response.setHeader("Content-Disposition", "attachment; filename=\"" + filename + "\""); + + var results = getAggregatedExportData(request); + + try (SXSSFWorkbook workbook = new SXSSFWorkbook(100); + ServletOutputStream outputStream = response.getOutputStream()) { + + Sheet sheet = workbook.createSheet("Dashboard Data"); + + String[] columns = {"Department", "URL", "Total Comments", "Language", "Section", "Theme"}; + var headerRow = sheet.createRow(0); + for (int i = 0; i < columns.length; i++) { + headerRow.createCell(i).setCellValue(columns[i]); + } + + int rowNum = 1; + for (Problem p : results) { + Row row = sheet.createRow(rowNum++); + row.createCell(0).setCellValue(resolveInstitutionName(p.getInstitution(), pageLang)); + row.createCell(1).setCellValue(p.getUrl()); + row.createCell(2).setCellValue(p.getUrlEntries()); + row.createCell(3).setCellValue(p.getLanguage()); + row.createCell(4).setCellValue(p.getSection()); + row.createCell(5).setCellValue(p.getTheme()); + + if (rowNum % 100 == 0) { + try { + ((SXSSFSheet) sheet).flushRows(100); + } catch (IOException e) { + LOG.error("Error flushing rows", e); + } + } + } - private List getAggregatedExportData(HttpServletRequest request) { - var criteria = buildExportCriteria(request); - var match = Aggregation.match(criteria); - var groupByUrl = Aggregation.group("url") - .first("url").as("url") - .first("institution").as("institution") - .first("language").as("language") - .first("section").as("section") - .first("theme").as("theme") - .count().as("urlEntries"); - var sortDesc = Aggregation.sort(Sort.Direction.DESC, "urlEntries"); - - return mongoTemplate.aggregate( - Aggregation.newAggregation(match, groupByUrl, sortDesc), - "problem", Problem.class).getMappedResults(); + workbook.write(outputStream); + } catch (Exception e) { + LOG.error("Error exporting Excel", e); + response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); } - - private String resolveInstitutionName(String institution, String lang) { - if (institution == null) return ""; - for (List variations : institutionMappings.values()) { - if (variations.contains(institution)) { - return variations.get("fr".equalsIgnoreCase(lang) ? 1 : 0); - } - } - return institution; + } + + @GetMapping("/dashboard/exportCSV") + public void exportCSV(HttpServletRequest request, HttpServletResponse response) + throws IOException { + String pageLang = (String) request.getSession().getAttribute("lang"); + String filename = buildExportFilename(pageLang, ".csv"); + response.setCharacterEncoding("UTF-8"); + response.setContentType("text/csv; charset=UTF-8"); + response.setHeader("Content-Disposition", "attachment; filename*=UTF-8''" + filename); + + var results = getAggregatedExportData(request); + + try (Writer writer = response.getWriter()) { + writer.write("\uFEFF"); + writer.write("Department,URL,Total Comments,Language,Section,Theme\n"); + + for (Problem p : results) { + writer.write( + String.format( + "%s,%s,%d,%s,%s,%s\n", + escapeCSV(resolveInstitutionName(p.getInstitution(), pageLang)), + escapeCSV(p.getUrl()), + p.getUrlEntries(), + escapeCSV(p.getLanguage()), + escapeCSV(p.getSection()), + escapeCSV(p.getTheme()))); + } } - - private Criteria buildExportCriteria(HttpServletRequest request) { - String language = request.getParameter("language"); - String department = request.getParameter("department"); - String comments = request.getParameter("comments"); - String theme = request.getParameter("theme"); - String section = request.getParameter("section"); - String url = request.getParameter("url"); - String startDate = request.getParameter("startDate"); - String endDate = request.getParameter("endDate"); - boolean error_keyword = "true".equals(request.getParameter("error_keyword")); - - var criteria = buildFilterCriteria(startDate, endDate, theme, section, language, url, department); - return applyRegexCriteria(criteria, comments, error_keyword); + } + + private List getAggregatedExportData(HttpServletRequest request) { + var criteria = buildExportCriteria(request); + var match = Aggregation.match(criteria); + var groupByUrl = + Aggregation.group("url") + .first("url") + .as("url") + .first("institution") + .as("institution") + .first("language") + .as("language") + .first("section") + .as("section") + .first("theme") + .as("theme") + .count() + .as("urlEntries"); + var sortDesc = Aggregation.sort(Sort.Direction.DESC, "urlEntries"); + + return mongoTemplate + .aggregate( + Aggregation.newAggregation(match, groupByUrl, sortDesc), "problem", Problem.class) + .getMappedResults(); + } + + private String resolveInstitutionName(String institution, String lang) { + if (institution == null) return ""; + for (List variations : institutionMappings.values()) { + if (variations.contains(institution)) { + return variations.get("fr".equalsIgnoreCase(lang) ? 1 : 0); + } } - - private String escapeCSV(String value) { - if (value == null) { - return ""; - } - String sanitized = value.replace("\t", " ").replace("\r", ""); - if (!sanitized.isEmpty()) { - char firstChar = sanitized.charAt(0); - if (firstChar == '=' || firstChar == '+' || firstChar == '-' || firstChar == '@') { - sanitized = "'" + sanitized; - } - } - return "\"" + sanitized.replace("\"", "\"\"") + "\""; + return institution; + } + + private Criteria buildExportCriteria(HttpServletRequest request) { + String language = request.getParameter("language"); + String department = request.getParameter("department"); + String comments = request.getParameter("comments"); + String theme = request.getParameter("theme"); + String section = request.getParameter("section"); + String url = request.getParameter("url"); + String startDate = request.getParameter("startDate"); + String endDate = request.getParameter("endDate"); + boolean error_keyword = "true".equals(request.getParameter("error_keyword")); + + var criteria = + buildFilterCriteria(startDate, endDate, theme, section, language, url, department); + return applyRegexCriteria(criteria, comments, error_keyword); + } + + private String escapeCSV(String value) { + if (value == null) { + return ""; } - - private String buildExportFilename(String lang, String extension) { - String prefix = "fr".equalsIgnoreCase(lang) ? "Outil_de_retroaction-" : "Page_feedback-"; - String date = LocalDate.now().format(DateTimeFormatter.ISO_LOCAL_DATE); - return prefix + date + extension; + String sanitized = value.replace("\t", " ").replace("\r", ""); + if (!sanitized.isEmpty()) { + char firstChar = sanitized.charAt(0); + if (firstChar == '=' || firstChar == '+' || firstChar == '-' || firstChar == '@') { + sanitized = "'" + sanitized; + } } - + return "\"" + sanitized.replace("\"", "\"\"") + "\""; + } + + private String buildExportFilename(String lang, String extension) { + String prefix = "fr".equalsIgnoreCase(lang) ? "Outil_de_retroaction-" : "Page_feedback-"; + String date = LocalDate.now().format(DateTimeFormatter.ISO_LOCAL_DATE); + return prefix + date + extension; + } } diff --git a/src/main/java/ca/gc/tbs/controller/ImportController.java b/src/main/java/ca/gc/tbs/controller/ImportController.java index d7ba030f..85a1545b 100644 --- a/src/main/java/ca/gc/tbs/controller/ImportController.java +++ b/src/main/java/ca/gc/tbs/controller/ImportController.java @@ -16,6 +16,7 @@ import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.servlet.View; import org.springframework.web.servlet.view.RedirectView; + // @Controller diff --git a/src/main/java/ca/gc/tbs/controller/LoginController.java b/src/main/java/ca/gc/tbs/controller/LoginController.java index 7745cc28..ad292add 100644 --- a/src/main/java/ca/gc/tbs/controller/LoginController.java +++ b/src/main/java/ca/gc/tbs/controller/LoginController.java @@ -2,9 +2,9 @@ import ca.gc.tbs.domain.User; import ca.gc.tbs.service.UserService; -import java.text.SimpleDateFormat; import jakarta.servlet.http.HttpServletRequest; import jakarta.validation.Valid; +import java.text.SimpleDateFormat; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; diff --git a/src/main/java/ca/gc/tbs/controller/ProblemController.java b/src/main/java/ca/gc/tbs/controller/ProblemController.java index b11e1626..81dc0c4e 100644 --- a/src/main/java/ca/gc/tbs/controller/ProblemController.java +++ b/src/main/java/ca/gc/tbs/controller/ProblemController.java @@ -7,6 +7,18 @@ import ca.gc.tbs.service.ProblemCacheService; import ca.gc.tbs.service.ProblemDateService; import ca.gc.tbs.service.UserService; +import jakarta.servlet.ServletOutputStream; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import jakarta.validation.Valid; +import java.io.IOException; +import java.io.Writer; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.*; +import java.util.regex.Pattern; +import java.util.stream.Collectors; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.xssf.streaming.SXSSFSheet; @@ -28,1117 +40,1138 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.servlet.ModelAndView; -import jakarta.servlet.ServletOutputStream; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; -import jakarta.validation.Valid; -import java.io.IOException; -import java.io.Writer; -import java.time.LocalDate; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeParseException; -import java.util.*; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - @Controller public class ProblemController { - private static final Logger LOG = LoggerFactory.getLogger(ProblemController.class); - - private final ProblemRepository problemRepository; - - private final ProblemDateService problemDateService; - - private final ErrorKeywordService errorKeywordService; - - private final UserService userService; - - private final ProblemCacheService problemCacheService; - - private final MongoTemplate mongoTemplate; - - private final JWTUtil jwtUtil; - - public ProblemController( - ProblemRepository problemRepository, - ProblemDateService problemDateService, - ErrorKeywordService errorKeywordService, - UserService userService, - ProblemCacheService problemCacheService, - MongoTemplate mongoTemplate, - JWTUtil jwtUtil) { - this.problemRepository = problemRepository; - this.problemDateService = problemDateService; - this.errorKeywordService = errorKeywordService; - this.userService = userService; - this.problemCacheService = problemCacheService; - this.mongoTemplate = mongoTemplate; - this.jwtUtil = jwtUtil; + private static final Logger LOG = LoggerFactory.getLogger(ProblemController.class); + + private final ProblemRepository problemRepository; + + private final ProblemDateService problemDateService; + + private final ErrorKeywordService errorKeywordService; + + private final UserService userService; + + private final ProblemCacheService problemCacheService; + + private final MongoTemplate mongoTemplate; + + private final JWTUtil jwtUtil; + + public ProblemController( + ProblemRepository problemRepository, + ProblemDateService problemDateService, + ErrorKeywordService errorKeywordService, + UserService userService, + ProblemCacheService problemCacheService, + MongoTemplate mongoTemplate, + JWTUtil jwtUtil) { + this.problemRepository = problemRepository; + this.problemDateService = problemDateService; + this.errorKeywordService = errorKeywordService; + this.userService = userService; + this.problemCacheService = problemCacheService; + this.mongoTemplate = mongoTemplate; + this.jwtUtil = jwtUtil; + } + + private static final Map> institutionMappings = new HashMap<>(); + private static final Map> sectionMappings = new HashMap<>(); + + static { + // Initialize section mappings + sectionMappings.put("disability", Arrays.asList("disability", "disability benefits")); + + // Initialize institution mappings + institutionMappings.put( + "AAFC", + Arrays.asList( + "AAFC", + "AAC", + "AGRICULTURE AND AGRI-FOOD CANADA", + "AGRICULTURE ET AGROALIMENTAIRE CANADA", + "AAFC / AAC")); + institutionMappings.put( + "ACOA", + Arrays.asList( + "ACOA", + "APECA", + "ATLANTIC CANADA OPPORTUNITIES AGENCY", + "AGENCE DE PROMOTION ÉCONOMIQUE DU CANADA ATLANTIQUE", + "ACOA / APECA")); + institutionMappings.put( + "ATSSC", + Arrays.asList( + "ATSSC", + "SCDATA", + "ADMINISTRATIVE TRIBUNALS SUPPORT SERVICE OF CANADA", + "SERVICE CANADIEN D'APPUI AUX TRIBUNAUX ADMINISTRATIFS", + "ATSSC / SCDATA")); + institutionMappings.put( + "CANNOR", + Arrays.asList( + "CANNOR", + "RNCAN", + "CANADIAN NORTHERN ECONOMIC DEVELOPMENT AGENCY", + "AGENCE CANADIENNE DE DÉVELOPPEMENT ÉCONOMIQUE DU NORD", + "CANNOR / RNCAN")); + institutionMappings.put( + "CATSA", + Arrays.asList( + "CATSA", + "ACSTA", + "CANADIAN AIR TRANSPORT SECURITY AUTHORITY", + "ADMINISTRATION CANADIENNE DE LA SÛRETÉ DU TRANSPORT AÉRIEN", + "CATSA / ACSTA")); + institutionMappings.put( + "CBSA", + Arrays.asList( + "CBSA", + "ASFC", + "CANADA BORDER SERVICES AGENCY", + "AGENCE DES SERVICES FRONTALIERS DU CANADA", + "CBSA / ASFC")); + institutionMappings.put( + "CCG", + Arrays.asList( + "CCG", "GCC", "CANADIAN COAST GUARD", "GARDE CÔTIÈRE CANADIENNE", "CCG / GCC")); + institutionMappings.put( + "CER", + Arrays.asList( + "CER", "REC", "CANADA ENERGY REGULATOR", "RÉGIE DE L'ÉNERGIE DU CANADA", "CER / REC")); + institutionMappings.put( + "CFIA", + Arrays.asList( + "CFIA", + "ACIA", + "CANADIAN FOOD INSPECTION AGENCY", + "AGENCE CANADIENNE D'INSPECTION DES ALIMENTS", + "CFIA / ACIA")); + institutionMappings.put( + "CGC", + Arrays.asList( + "CGC", "CANADIAN GRAIN COMMISSION", "COMMISSION CANADIENNE DES GRAINS", "CGC")); + institutionMappings.put( + "CIHR", + Arrays.asList( + "CIHR", + "IRSC", + "CANADIAN INSTITUTES OF HEALTH RESEARCH", + "INSTITUTS DE RECHERCHE EN SANTÉ DU CANADA", + "CIHR / IRSC")); + institutionMappings.put( + "CIPO", + Arrays.asList( + "CIPO", + "OPIC", + "CANADIAN INTELLECTUAL PROPERTY OFFICE", + "OFFICE DE LA PROPRIÉTÉ INTELLECTUELLE DU CANADA", + "CIPO / OPIC")); + institutionMappings.put( + "CIRNAC", + Arrays.asList( + "CIRNAC", + "RCAANC", + "CROWN-INDIGENOUS RELATIONS AND NORTHERN AFFAIRS CANADA", + "RELATIONS COURONNE-AUTOCHTONES ET AFFAIRES DU NORD CANADA", + "CIRNAC / RCAANC")); + institutionMappings.put( + "CRA", + Arrays.asList( + "CRA", "ARC", "CANADA REVENUE AGENCY", "AGENCE DU REVENU DU CANADA", "CRA / ARC")); + institutionMappings.put( + "CRTC", + Arrays.asList( + "CRTC", + "CRTC", + "CANADIAN RADIO-TELEVISION AND TELECOMMUNICATIONS COMMISSION", + "CONSEIL DE LA RADIODIFFUSION ET DES TÉLÉCOMMUNICATIONS CANADIENNES", + "CRTC / CRTC")); + institutionMappings.put( + "CSA", + Arrays.asList( + "CSA", "ASC", "CANADIAN SPACE AGENCY", "AGENCE SPATIALE CANADIENNE", "CSA / ASC")); + institutionMappings.put( + "CSC", + Arrays.asList( + "CSC", + "SCC", + "CORRECTIONAL SERVICE CANADA", + "SERVICE CORRECTIONNEL CANADA", + "CSC / SCC")); + institutionMappings.put( + "CSE", + Arrays.asList( + "CSE", + "CST", + "COMMUNICATIONS SECURITY ESTABLISHMENT", + "CENTRE DE LA SÉCURITÉ DES TÉLÉCOMMUNICATIONS", + "CSE / CST")); + institutionMappings.put( + "CSEC", + Arrays.asList( + "CSEC", + "CSTC", + "COMMUNICATIONS SECURITY ESTABLISHMENT CANADA", + "CENTRE DE LA SÉCURITÉ DES TÉLÉCOMMUNICATIONS CANADA", + "CSEC / CSTC")); + institutionMappings.put( + "CSPS", + Arrays.asList( + "CSPS", + "EFPC", + "CANADA SCHOOL OF PUBLIC SERVICE", + "ÉCOLE DE LA FONCTION PUBLIQUE DU CANADA", + "CSPS / EFPC")); + institutionMappings.put( + "DFO", + Arrays.asList( + "DFO", + "MPO", + "FISHERIES AND OCEANS CANADA", + "PÊCHES ET OCÉANS CANADA", + "DFO / MPO", + "GOVERNMENT OF CANADA, FISHERIES AND OCEANS CANADA, COMMUNICATIONS BRANCH")); + institutionMappings.put( + "DND", Arrays.asList("DND", "MDN", "NATIONAL DEFENCE", "DÉFENSE NATIONALE", "DND / MDN")); + institutionMappings.put( + "ECCC", + Arrays.asList( + "ECCC", + "ECCC", + "ENVIRONMENT AND CLIMATE CHANGE CANADA", + "ENVIRONNEMENT ET CHANGEMENT CLIMATIQUE CANADA", + "ECCC / ECCC")); + institutionMappings.put( + "ESDC", + Arrays.asList( + "ESDC", + "EDSC", + "EMPLOYMENT AND SOCIAL DEVELOPMENT CANADA", + "EMPLOI ET DÉVELOPPEMENT SOCIAL CANADA", + "ESDC / EDSC")); + institutionMappings.put( + "FCAC", + Arrays.asList( + "FCAC", + "ACFC", + "FINANCIAL CONSUMER AGENCY OF CANADA", + "AGENCE DE LA CONSOMMATION EN MATIÈRE FINANCIÈRE DU CANADA", + "FCAC / ACFC")); + institutionMappings.put( + "FIN", + Arrays.asList( + "FIN", + "FIN", + "FINANCE CANADA", + "MINISTÈRE DES FINANCES CANADA", + "DEPARTMENT OF FINANCE CANADA", + "GOVERNMENT OF CANADA, DEPARTMENT OF FINANCE", + "MINISTÈRE DES FINANCES", + "FIN / FIN")); + institutionMappings.put( + "GAC", + Arrays.asList( + "GAC", "AMC", "GLOBAL AFFAIRS CANADA", "AFFAIRES MONDIALES CANADA", "GAC / AMC")); + institutionMappings.put( + "HC", Arrays.asList("HC", "SC", "HEALTH CANADA", "SANTÉ CANADA", "HC / SC")); + institutionMappings.put( + "HICC", + Arrays.asList( + "HICC", + "LICC", + "HOUSING, INFRASTRUCTURE AND COMMUNITIES CANADA", + "LOGEMENT, INFRASTRUCTURES ET COLLECTIVITÉS CANADA", + "HICC / LICC")); + institutionMappings.put( + "INFC", + Arrays.asList( + "INFC", "INFC", "INFRASTRUCTURE CANADA", "INFRASTRUCTURE CANADA", "INFC / INFC")); + institutionMappings.put( + "IOGC", + Arrays.asList( + "IOGC", + "BPGI", + "INDIAN OIL AND GAS CANADA", + "BUREAU DU PÉTROLE ET DU GAZ DES INDIENS", + "IOGC / BPGI")); + institutionMappings.put( + "IRCC", + Arrays.asList( + "IRCC", + "IRCC", + "IMMIGRATION, REFUGEES AND CITIZENSHIP CANADA", + "IMMIGRATION, RÉFUGIÉS ET CITOYENNETÉ CANADA", + "IRCC / IRCC")); + institutionMappings.put( + "ISC", + Arrays.asList( + "ISC", + "SAC", + "INDIGENOUS SERVICES CANADA", + "SERVICES AUX AUTOCHTONES CANADA", + "ISC / SAC")); + institutionMappings.put( + "ISED", + Arrays.asList( + "ISED", + "ISDE", + "INNOVATION, SCIENCE AND ECONOMIC DEVELOPMENT CANADA", + "INNOVATION, SCIENCES ET DÉVELOPPEMENT ÉCONOMIQUE CANADA", + "ISED / ISDE")); + institutionMappings.put( + "JUS", + Arrays.asList( + "JUS", "JUS", "JUSTICE CANADA", "MINISTÈRE DE LA JUSTICE CANADA", "JUS / JUS")); + institutionMappings.put( + "LAC", + Arrays.asList( + "LAC", + "BAC", + "LIBRARY AND ARCHIVES CANADA", + "BIBLIOTHÈQUE ET ARCHIVES CANADA", + "LAC / BAC")); + institutionMappings.put( + "NFB", + Arrays.asList("NFB", "ONF", "NATIONAL FILM BOARD", "OFFICE NATIONAL DU FILM", "NFB / ONF")); + institutionMappings.put( + "NRC", + Arrays.asList( + "NRC", + "CNRC", + "NATIONAL RESEARCH COUNCIL", + "CONSEIL NATIONAL DE RECHERCHES CANADA", + "NRC / CNRC")); + institutionMappings.put( + "NRCAN", + Arrays.asList( + "NRCAN", + "RNCAN", + "NATURAL RESOURCES CANADA", + "RESSOURCES NATURELLES CANADA", + "NRCAN / RNCAN")); + institutionMappings.put( + "NSERC", + Arrays.asList( + "NSERC", + "CRSNG", + "NATURAL SCIENCES AND ENGINEERING RESEARCH CANADA", + "CONSEIL DE RECHERCHES EN SCIENCES NATURELLES ET EN GÉNIE DU CANADA", + "NSERC / CRSNG")); + institutionMappings.put( + "OMBDNDCAF", + Arrays.asList( + "OMBDNDCAF", + "OMBMDNFAC", + "DND / CAF OMBUDSMAN", + "OMBUDSMAN DU MDN / FAC", + "OFFICE OF THE NATIONAL DEFENCE AND CANADIAN ARMED FORCES OMBUDSMAN", + "BUREAU DE L'OMBUDSMAN DE LA DÉFENSE NATIONALE ET DES FORCES ARMÉES CANADIENNES", + "OMBDNDCAF / OMBMDNFAC")); + institutionMappings.put( + "OSB", + Arrays.asList( + "OSB", + "BSF", + "SUPERINTENDENT OF BANKRUPTCY CANADA", + "BUREAU DU SURINTENDANT DES FAILLITES CANADA", + "OSB / BSF")); + institutionMappings.put( + "PBC", + Arrays.asList( + "PBC", + "CLCC", + "PAROLE BOARD OF CANADA", + "COMMISSION DES LIBÉRATIONS CONDITIONNELLES DU CANADA", + "PBC / CLCC")); + institutionMappings.put( + "PC", Arrays.asList("PC", "PC", "PARCS CANADA", "PARKS CANADA", "PC / PC")); + institutionMappings.put( + "PCH", + Arrays.asList("PCH", "PCH", "CANADIAN HERITAGE", "PATRIMOINE CANADIEN", "PCH / PCH")); + institutionMappings.put( + "PCO", + Arrays.asList( + "PCO", "BCP", "PRIVY COUNCIL OFFICE", "BUREAU DU CONSEIL PRIVÉ", "PCO / BCP")); + institutionMappings.put( + "PHAC", + Arrays.asList( + "PHAC", + "ASPC", + "PUBLIC HEALTH AGENCY OF CANADA", + "AGENCE DE LA SANTÉ PUBLIQUE DU CANADA", + "PHAC / ASPC")); + institutionMappings.put( + "PS", + Arrays.asList("PS", "SP", "PUBLIC SAFETY CANADA", "SÉCURITÉ PUBLIQUE CANADA", "PS / SP")); + institutionMappings.put( + "PSC", + Arrays.asList( + "PSC", + "CFP", + "PUBLIC SERVICE COMMISSION OF CANADA", + "COMMISSION DE LA FONCTION PUBLIQUE DU CANADA", + "PSC / CFP")); + institutionMappings.put( + "PSPC", + Arrays.asList( + "PSPC", + "SPAC", + "PUBLIC SERVICES AND PROCUREMENT CANADA", + "SERVICES PUBLICS ET APPROVISIONNEMENT CANADA", + "GOUVERNEMENT DU CANADA, SERVICES PUBLICS ET APPROVISIONNEMENT CANADA", + "GOVERNMENT OF CANADA, PUBLIC SERVICES AND PROCUREMENT CANADA", + "PSPC / SPAC")); + institutionMappings.put( + "RCMP", + Arrays.asList( + "RCMP", + "GRC", + "ROYAL CANADIAN MOUNTED POLICE", + "GENDARMERIE ROYALE DU CANADA", + "RCMP / GRC")); + institutionMappings.put( + "SC", Arrays.asList("SC", "SC", "SERVICE CANADA", "SERVICE CANADA", "SC / SC")); + institutionMappings.put( + "SSC", + Arrays.asList( + "SSC", "PSC", "SHARED SERVICES CANADA", "SERVICES PARTAGÉS CANADA", "SSC / PSC")); + institutionMappings.put( + "SSHRC", + Arrays.asList( + "SSHRC", + "CRSH", + "SOCIAL SCIENCES AND HUMANITIES RESEARCH COUNCIL", + "CONSEIL DE RECHERCHES EN SCIENCES HUMAINES", + "SSHRC / CRSH")); + institutionMappings.put( + "SST", + Arrays.asList( + "SST", + "TSS", + "SOCIAL SECURITY TRIBUNAL OF CANADA", + "TRIBUNAL DE LA SÉCURITÉ SOCIALE DU CANADA", + "SST / TSS")); + institutionMappings.put( + "STATCAN", + Arrays.asList( + "STATCAN", "STATCAN", "STATISTICS CANADA", "STATISTIQUE CANADA", "STATCAN / STATCAN")); + institutionMappings.put( + "TBS", + Arrays.asList( + "TBS", + "SCT", + "TREASURY BOARD OF CANADA SECRETARIAT", + "SECRÉTARIAT DU CONSEIL DU TRÉSOR DU CANADA", + "TBS / SCT")); + institutionMappings.put( + "TC", Arrays.asList("TC", "TC", "TRANSPORT CANADA", "TRANSPORTS CANADA", "TC / TC")); + institutionMappings.put( + "VAC", + Arrays.asList( + "VAC", "ACC", "VETERANS AFFAIRS CANADA", "ANCIENS COMBATTANTS CANADA", "VAC / ACC")); + institutionMappings.put( + "WAGE", + Arrays.asList( + "WAGE", + "FEGC", + "WOMEN AND GENDER EQUALITY CANADA", + "FEMMES ET ÉGALITÉ DES GENRES CANADA", + "WAGE / FEGC")); + institutionMappings.put( + "WD", + Arrays.asList( + "WD", + "DEO", + "WESTERN ECONOMIC DIVERSIFICATION CANADA", + "DIVERSIFICATION DE L'ÉCONOMIE DE L'OUEST CANADA", + "WD / DEO")); + } + + @GetMapping("/pageTitles") + @ResponseBody + public List getPageTitles( + @RequestParam(name = "search", required = false) String search) { + if (search != null && !search.isEmpty()) { + // Use the new repository method to filter page titles based on the search term + return problemRepository.findPageTitlesBySearch(search); + } else { + // Return all page titles if no search term is provided + return problemRepository.findDistinctPageNames(); } - - private static final Map> institutionMappings = new HashMap<>(); - private static final Map> sectionMappings = new HashMap<>(); - - static { - // Initialize section mappings - sectionMappings.put("disability", Arrays.asList("disability", "disability benefits")); - - // Initialize institution mappings - institutionMappings.put( - "AAFC", - Arrays.asList( - "AAFC", - "AAC", - "AGRICULTURE AND AGRI-FOOD CANADA", - "AGRICULTURE ET AGROALIMENTAIRE CANADA", - "AAFC / AAC")); - institutionMappings.put( - "ACOA", - Arrays.asList( - "ACOA", - "APECA", - "ATLANTIC CANADA OPPORTUNITIES AGENCY", - "AGENCE DE PROMOTION ÉCONOMIQUE DU CANADA ATLANTIQUE", - "ACOA / APECA")); - institutionMappings.put( - "ATSSC", - Arrays.asList( - "ATSSC", - "SCDATA", - "ADMINISTRATIVE TRIBUNALS SUPPORT SERVICE OF CANADA", - "SERVICE CANADIEN D'APPUI AUX TRIBUNAUX ADMINISTRATIFS", - "ATSSC / SCDATA")); - institutionMappings.put( - "CANNOR", - Arrays.asList( - "CANNOR", - "RNCAN", - "CANADIAN NORTHERN ECONOMIC DEVELOPMENT AGENCY", - "AGENCE CANADIENNE DE DÉVELOPPEMENT ÉCONOMIQUE DU NORD", - "CANNOR / RNCAN")); - institutionMappings.put( - "CATSA", - Arrays.asList( - "CATSA", - "ACSTA", - "CANADIAN AIR TRANSPORT SECURITY AUTHORITY", - "ADMINISTRATION CANADIENNE DE LA SÛRETÉ DU TRANSPORT AÉRIEN", - "CATSA / ACSTA")); - institutionMappings.put( - "CBSA", - Arrays.asList( - "CBSA", - "ASFC", - "CANADA BORDER SERVICES AGENCY", - "AGENCE DES SERVICES FRONTALIERS DU CANADA", - "CBSA / ASFC")); - institutionMappings.put( - "CCG", - Arrays.asList( - "CCG", "GCC", "CANADIAN COAST GUARD", "GARDE CÔTIÈRE CANADIENNE", "CCG / GCC")); - institutionMappings.put( - "CER", - Arrays.asList( - "CER", "REC", "CANADA ENERGY REGULATOR", "RÉGIE DE L'ÉNERGIE DU CANADA", "CER / REC")); - institutionMappings.put( - "CFIA", - Arrays.asList( - "CFIA", - "ACIA", - "CANADIAN FOOD INSPECTION AGENCY", - "AGENCE CANADIENNE D'INSPECTION DES ALIMENTS", - "CFIA / ACIA")); - institutionMappings.put( - "CGC", - Arrays.asList( - "CGC", - "CANADIAN GRAIN COMMISSION", - "COMMISSION CANADIENNE DES GRAINS", - "CGC")); - institutionMappings.put( - "CIHR", - Arrays.asList( - "CIHR", - "IRSC", - "CANADIAN INSTITUTES OF HEALTH RESEARCH", - "INSTITUTS DE RECHERCHE EN SANTÉ DU CANADA", - "CIHR / IRSC")); - institutionMappings.put( - "CIPO", - Arrays.asList( - "CIPO", - "OPIC", - "CANADIAN INTELLECTUAL PROPERTY OFFICE", - "OFFICE DE LA PROPRIÉTÉ INTELLECTUELLE DU CANADA", - "CIPO / OPIC")); - institutionMappings.put( - "CIRNAC", - Arrays.asList( - "CIRNAC", - "RCAANC", - "CROWN-INDIGENOUS RELATIONS AND NORTHERN AFFAIRS CANADA", - "RELATIONS COURONNE-AUTOCHTONES ET AFFAIRES DU NORD CANADA", - "CIRNAC / RCAANC")); - institutionMappings.put( - "CRA", - Arrays.asList( - "CRA", "ARC", "CANADA REVENUE AGENCY", "AGENCE DU REVENU DU CANADA", "CRA / ARC")); - institutionMappings.put( - "CRTC", - Arrays.asList( - "CRTC", - "CRTC", - "CANADIAN RADIO-TELEVISION AND TELECOMMUNICATIONS COMMISSION", - "CONSEIL DE LA RADIODIFFUSION ET DES TÉLÉCOMMUNICATIONS CANADIENNES", - "CRTC / CRTC")); - institutionMappings.put( - "CSA", - Arrays.asList( - "CSA", "ASC", "CANADIAN SPACE AGENCY", "AGENCE SPATIALE CANADIENNE", "CSA / ASC")); - institutionMappings.put( - "CSC", - Arrays.asList( - "CSC", - "SCC", - "CORRECTIONAL SERVICE CANADA", - "SERVICE CORRECTIONNEL CANADA", - "CSC / SCC")); - institutionMappings.put( - "CSE", - Arrays.asList( - "CSE", - "CST", - "COMMUNICATIONS SECURITY ESTABLISHMENT", - "CENTRE DE LA SÉCURITÉ DES TÉLÉCOMMUNICATIONS", - "CSE / CST")); - institutionMappings.put( - "CSEC", - Arrays.asList( - "CSEC", - "CSTC", - "COMMUNICATIONS SECURITY ESTABLISHMENT CANADA", - "CENTRE DE LA SÉCURITÉ DES TÉLÉCOMMUNICATIONS CANADA", - "CSEC / CSTC")); - institutionMappings.put( - "CSPS", - Arrays.asList( - "CSPS", - "EFPC", - "CANADA SCHOOL OF PUBLIC SERVICE", - "ÉCOLE DE LA FONCTION PUBLIQUE DU CANADA", - "CSPS / EFPC")); - institutionMappings.put( - "DFO", - Arrays.asList( - "DFO", "MPO", "FISHERIES AND OCEANS CANADA", "PÊCHES ET OCÉANS CANADA", "DFO / MPO", "GOVERNMENT OF CANADA, FISHERIES AND OCEANS CANADA, COMMUNICATIONS BRANCH")); - institutionMappings.put( - "DND", Arrays.asList("DND", "MDN", "NATIONAL DEFENCE", "DÉFENSE NATIONALE", "DND / MDN")); - institutionMappings.put( - "ECCC", - Arrays.asList( - "ECCC", - "ECCC", - "ENVIRONMENT AND CLIMATE CHANGE CANADA", - "ENVIRONNEMENT ET CHANGEMENT CLIMATIQUE CANADA", - "ECCC / ECCC")); - institutionMappings.put( - "ESDC", - Arrays.asList( - "ESDC", - "EDSC", - "EMPLOYMENT AND SOCIAL DEVELOPMENT CANADA", - "EMPLOI ET DÉVELOPPEMENT SOCIAL CANADA", - "ESDC / EDSC")); - institutionMappings.put( - "FCAC", - Arrays.asList( - "FCAC", - "ACFC", - "FINANCIAL CONSUMER AGENCY OF CANADA", - "AGENCE DE LA CONSOMMATION EN MATIÈRE FINANCIÈRE DU CANADA", - "FCAC / ACFC")); - institutionMappings.put( - "FIN", - Arrays.asList( - "FIN", - "FIN", - "FINANCE CANADA", - "MINISTÈRE DES FINANCES CANADA", - "DEPARTMENT OF FINANCE CANADA", - "GOVERNMENT OF CANADA, DEPARTMENT OF FINANCE", - "MINISTÈRE DES FINANCES", - "FIN / FIN")); - institutionMappings.put( - "GAC", - Arrays.asList( - "GAC", "AMC", "GLOBAL AFFAIRS CANADA", "AFFAIRES MONDIALES CANADA", "GAC / AMC")); - institutionMappings.put( - "HC", Arrays.asList("HC", "SC", "HEALTH CANADA", "SANTÉ CANADA", "HC / SC")); - institutionMappings.put( - "HICC", Arrays.asList( - "HICC", "LICC", "HOUSING, INFRASTRUCTURE AND COMMUNITIES CANADA", "LOGEMENT, INFRASTRUCTURES ET COLLECTIVITÉS CANADA", "HICC / LICC")); - institutionMappings.put( - "INFC", - Arrays.asList( - "INFC", "INFC", "INFRASTRUCTURE CANADA", "INFRASTRUCTURE CANADA", "INFC / INFC")); - institutionMappings.put( - "IOGC", - Arrays.asList( - "IOGC", - "BPGI", - "INDIAN OIL AND GAS CANADA", - "BUREAU DU PÉTROLE ET DU GAZ DES INDIENS", - "IOGC / BPGI")); - institutionMappings.put( - "IRCC", - Arrays.asList( - "IRCC", - "IRCC", - "IMMIGRATION, REFUGEES AND CITIZENSHIP CANADA", - "IMMIGRATION, RÉFUGIÉS ET CITOYENNETÉ CANADA", - "IRCC / IRCC")); - institutionMappings.put( - "ISC", - Arrays.asList( - "ISC", - "SAC", - "INDIGENOUS SERVICES CANADA", - "SERVICES AUX AUTOCHTONES CANADA", - "ISC / SAC")); - institutionMappings.put( - "ISED", - Arrays.asList( - "ISED", - "ISDE", - "INNOVATION, SCIENCE AND ECONOMIC DEVELOPMENT CANADA", - "INNOVATION, SCIENCES ET DÉVELOPPEMENT ÉCONOMIQUE CANADA", - "ISED / ISDE")); - institutionMappings.put( - "JUS", - Arrays.asList( - "JUS", "JUS", "JUSTICE CANADA", "MINISTÈRE DE LA JUSTICE CANADA", "JUS / JUS")); - institutionMappings.put( - "LAC", - Arrays.asList( - "LAC", - "BAC", - "LIBRARY AND ARCHIVES CANADA", - "BIBLIOTHÈQUE ET ARCHIVES CANADA", - "LAC / BAC")); - institutionMappings.put( - "NFB", - Arrays.asList("NFB", "ONF", "NATIONAL FILM BOARD", "OFFICE NATIONAL DU FILM", "NFB / ONF")); - institutionMappings.put( - "NRC", - Arrays.asList( - "NRC", - "CNRC", - "NATIONAL RESEARCH COUNCIL", - "CONSEIL NATIONAL DE RECHERCHES CANADA", - "NRC / CNRC")); - institutionMappings.put( - "NRCAN", - Arrays.asList( - "NRCAN", - "RNCAN", - "NATURAL RESOURCES CANADA", - "RESSOURCES NATURELLES CANADA", - "NRCAN / RNCAN")); - institutionMappings.put( - "NSERC", - Arrays.asList( - "NSERC", - "CRSNG", - "NATURAL SCIENCES AND ENGINEERING RESEARCH CANADA", - "CONSEIL DE RECHERCHES EN SCIENCES NATURELLES ET EN GÉNIE DU CANADA", - "NSERC / CRSNG")); - institutionMappings.put( - "OMBDNDCAF", - Arrays.asList( - "OMBDNDCAF", - "OMBMDNFAC", - "DND / CAF OMBUDSMAN", - "OMBUDSMAN DU MDN / FAC", - "OFFICE OF THE NATIONAL DEFENCE AND CANADIAN ARMED FORCES OMBUDSMAN", - "BUREAU DE L'OMBUDSMAN DE LA DÉFENSE NATIONALE ET DES FORCES ARMÉES CANADIENNES", - "OMBDNDCAF / OMBMDNFAC")); - institutionMappings.put( - "OSB", - Arrays.asList( - "OSB", - "BSF", - "SUPERINTENDENT OF BANKRUPTCY CANADA", - "BUREAU DU SURINTENDANT DES FAILLITES CANADA", - "OSB / BSF")); - institutionMappings.put( - "PBC", - Arrays.asList( - "PBC", - "CLCC", - "PAROLE BOARD OF CANADA", - "COMMISSION DES LIBÉRATIONS CONDITIONNELLES DU CANADA", - "PBC / CLCC")); - institutionMappings.put( - "PC", Arrays.asList("PC", "PC", "PARCS CANADA", "PARKS CANADA", "PC / PC")); - institutionMappings.put( - "PCH", - Arrays.asList("PCH", "PCH", "CANADIAN HERITAGE", "PATRIMOINE CANADIEN", "PCH / PCH")); - institutionMappings.put( - "PCO", - Arrays.asList( - "PCO", "BCP", "PRIVY COUNCIL OFFICE", "BUREAU DU CONSEIL PRIVÉ", "PCO / BCP")); - institutionMappings.put( - "PHAC", - Arrays.asList( - "PHAC", - "ASPC", - "PUBLIC HEALTH AGENCY OF CANADA", - "AGENCE DE LA SANTÉ PUBLIQUE DU CANADA", - "PHAC / ASPC")); - institutionMappings.put( - "PS", - Arrays.asList("PS", "SP", "PUBLIC SAFETY CANADA", "SÉCURITÉ PUBLIQUE CANADA", "PS / SP")); - institutionMappings.put( - "PSC", - Arrays.asList( - "PSC", - "CFP", - "PUBLIC SERVICE COMMISSION OF CANADA", - "COMMISSION DE LA FONCTION PUBLIQUE DU CANADA", - "PSC / CFP")); - institutionMappings.put( - "PSPC", - Arrays.asList( - "PSPC", - "SPAC", - "PUBLIC SERVICES AND PROCUREMENT CANADA", - "SERVICES PUBLICS ET APPROVISIONNEMENT CANADA", - "GOUVERNEMENT DU CANADA, SERVICES PUBLICS ET APPROVISIONNEMENT CANADA", - "GOVERNMENT OF CANADA, PUBLIC SERVICES AND PROCUREMENT CANADA", - "PSPC / SPAC")); - institutionMappings.put( - "RCMP", - Arrays.asList( - "RCMP", - "GRC", - "ROYAL CANADIAN MOUNTED POLICE", - "GENDARMERIE ROYALE DU CANADA", - "RCMP / GRC")); - institutionMappings.put( - "SC", Arrays.asList("SC", "SC", "SERVICE CANADA", "SERVICE CANADA", "SC / SC")); - institutionMappings.put( - "SSC", - Arrays.asList( - "SSC", "PSC", "SHARED SERVICES CANADA", "SERVICES PARTAGÉS CANADA", "SSC / PSC")); - institutionMappings.put( - "SSHRC", - Arrays.asList( - "SSHRC", - "CRSH", - "SOCIAL SCIENCES AND HUMANITIES RESEARCH COUNCIL", - "CONSEIL DE RECHERCHES EN SCIENCES HUMAINES", - "SSHRC / CRSH")); - institutionMappings.put( - "SST", - Arrays.asList( - "SST", - "TSS", - "SOCIAL SECURITY TRIBUNAL OF CANADA", - "TRIBUNAL DE LA SÉCURITÉ SOCIALE DU CANADA", - "SST / TSS")); - institutionMappings.put( - "STATCAN", - Arrays.asList( - "STATCAN", "STATCAN", "STATISTICS CANADA", "STATISTIQUE CANADA", "STATCAN / STATCAN")); - institutionMappings.put( - "TBS", - Arrays.asList( - "TBS", - "SCT", - "TREASURY BOARD OF CANADA SECRETARIAT", - "SECRÉTARIAT DU CONSEIL DU TRÉSOR DU CANADA", - "TBS / SCT")); - institutionMappings.put( - "TC", Arrays.asList("TC", "TC", "TRANSPORT CANADA", "TRANSPORTS CANADA", "TC / TC")); - institutionMappings.put( - "VAC", - Arrays.asList( - "VAC", "ACC", "VETERANS AFFAIRS CANADA", "ANCIENS COMBATTANTS CANADA", "VAC / ACC")); - institutionMappings.put( - "WAGE", - Arrays.asList( - "WAGE", - "FEGC", - "WOMEN AND GENDER EQUALITY CANADA", - "FEMMES ET ÉGALITÉ DES GENRES CANADA", - "WAGE / FEGC")); - institutionMappings.put( - "WD", - Arrays.asList( - "WD", - "DEO", - "WESTERN ECONOMIC DIVERSIFICATION CANADA", - "DIVERSIFICATION DE L'ÉCONOMIE DE L'OUEST CANADA", - "WD / DEO")); + } + + @GetMapping("/api/problems") + public ResponseEntity getProblemsJson( + @RequestParam Map requestParams, + @RequestParam(required = false) String startDate, + @RequestParam(required = false) String endDate, + @RequestParam(required = false) String processedStartDate, + @RequestParam(required = false) String processedEndDate, + @RequestParam(required = false) String institution, + @RequestParam(required = false) String url, + @RequestHeader(name = "Authorization") String authorizationHeader) { + String token = null; + String userName = null; + + if (authorizationHeader != null && authorizationHeader.startsWith("Bearer ")) { + token = authorizationHeader.substring(7); + userName = jwtUtil.extractUsername(token); } - @GetMapping("/pageTitles") - @ResponseBody - public List getPageTitles( - @RequestParam(name = "search", required = false) String search) { - if (search != null && !search.isEmpty()) { - // Use the new repository method to filter page titles based on the search term - return problemRepository.findPageTitlesBySearch(search); - } else { - // Return all page titles if no search term is provided - return problemRepository.findDistinctPageNames(); - } + if (userName != null) { + var user = userService.findUserByEmail(userName); + if (!userService.isAdmin(user) && !userService.isAPI(user)) { + return ResponseEntity.status(HttpStatus.FORBIDDEN) + .body("Access denied. Only API users & Admins can access this endpoint."); + } + } else { + return ResponseEntity.status(HttpStatus.UNAUTHORIZED) + .body("Authorization header is missing or invalid."); } + Set validParams = + new HashSet<>( + Arrays.asList( + "startDate", + "endDate", + "processedStartDate", + "processedEndDate", + "institution", + "url", + "authorizationHeader")); + + for (String param : requestParams.keySet()) { + if (!validParams.contains(param)) { + var errorResponse = new HashMap(); + errorResponse.put("error", "Invalid parameter: " + param); + return ResponseEntity.badRequest().body(errorResponse); + } + } - @GetMapping("/api/problems") - public ResponseEntity getProblemsJson( - @RequestParam Map requestParams, - @RequestParam(required = false) String startDate, - @RequestParam(required = false) String endDate, - @RequestParam(required = false) String processedStartDate, - @RequestParam(required = false) String processedEndDate, - @RequestParam(required = false) String institution, - @RequestParam(required = false) String url, - @RequestHeader(name = "Authorization") String authorizationHeader) { - String token = null; - String userName = null; - - if (authorizationHeader != null && authorizationHeader.startsWith("Bearer ")) { - token = authorizationHeader.substring(7); - userName = jwtUtil.extractUsername(token); - } + var criteria = new Criteria("processed").is("true"); + var dateFormat = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - if (userName != null) { - var user = userService.findUserByEmail(userName); - if (!userService.isAdmin(user) && !userService.isAPI(user)) { - return ResponseEntity.status(HttpStatus.FORBIDDEN) - .body("Access denied. Only API users & Admins can access this endpoint."); - } - } else { - return ResponseEntity.status(HttpStatus.UNAUTHORIZED) - .body("Authorization header is missing or invalid."); - } - - Set validParams = - new HashSet<>( - Arrays.asList( - "startDate", - "endDate", - "processedStartDate", - "processedEndDate", - "institution", - "url", - "authorizationHeader")); - - for (String param : requestParams.keySet()) { - if (!validParams.contains(param)) { - var errorResponse = new HashMap(); - errorResponse.put("error", "Invalid parameter: " + param); - return ResponseEntity.badRequest().body(errorResponse); - } - } - - var criteria = new Criteria("processed").is("true"); - var dateFormat = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + // Ensure only one type of date filter is used + if ((startDate != null || endDate != null) + && (processedStartDate != null || processedEndDate != null)) { + Map errorResponse = new HashMap<>(); + errorResponse.put( + "error", "You can only filter by normal date range or processed date range, not both."); + return ResponseEntity.badRequest().body(errorResponse); + } - // Ensure only one type of date filter is used - if ((startDate != null || endDate != null) - && (processedStartDate != null || processedEndDate != null)) { + // Validate and apply normal date range filter + if (startDate != null || endDate != null) { + if (startDate == null || endDate == null) { + Map errorResponse = new HashMap<>(); + errorResponse.put("error", "Both startDate and endDate are required."); + return ResponseEntity.badRequest().body(errorResponse); + } else { + try { + var start = LocalDate.parse(startDate, dateFormat); + var end = LocalDate.parse(endDate, dateFormat); + if (end.isBefore(start)) { Map errorResponse = new HashMap<>(); - errorResponse.put( - "error", "You can only filter by normal date range or processed date range, not both."); + errorResponse.put("error", "endDate must be greater than or equal to startDate."); return ResponseEntity.badRequest().body(errorResponse); + } + criteria.and("problemDate").gte(startDate).lte(endDate); + } catch (DateTimeParseException e) { + Map errorResponse = new HashMap<>(); + errorResponse.put("error", "Invalid date format. Please use yyyy-MM-dd."); + return ResponseEntity.badRequest().body(errorResponse); } + } + } - // Validate and apply normal date range filter - if (startDate != null || endDate != null) { - if (startDate == null || endDate == null) { - Map errorResponse = new HashMap<>(); - errorResponse.put("error", "Both startDate and endDate are required."); - return ResponseEntity.badRequest().body(errorResponse); - } else { - try { - var start = LocalDate.parse(startDate, dateFormat); - var end = LocalDate.parse(endDate, dateFormat); - if (end.isBefore(start)) { - Map errorResponse = new HashMap<>(); - errorResponse.put("error", "endDate must be greater than or equal to startDate."); - return ResponseEntity.badRequest().body(errorResponse); - } - criteria.and("problemDate").gte(startDate).lte(endDate); - } catch (DateTimeParseException e) { - Map errorResponse = new HashMap<>(); - errorResponse.put("error", "Invalid date format. Please use yyyy-MM-dd."); - return ResponseEntity.badRequest().body(errorResponse); - } - } - } - - // Validate and apply processed date range filter - if (processedStartDate != null || processedEndDate != null) { - if (processedStartDate == null || processedEndDate == null) { - Map errorResponse = new HashMap<>(); - errorResponse.put("error", "Both processedStartDate and processedEndDate are required."); - return ResponseEntity.badRequest().body(errorResponse); - } else { - try { - var processedStart = LocalDate.parse(processedStartDate, dateFormat); - var processedEnd = LocalDate.parse(processedEndDate, dateFormat); - if (processedEnd.isBefore(processedStart)) { - Map errorResponse = new HashMap<>(); - errorResponse.put( - "error", "processedEndDate must be greater than or equal to processedStartDate."); - return ResponseEntity.badRequest().body(errorResponse); - } - criteria.and("processedDate").gte(processedStartDate).lte(processedEndDate); - } catch (DateTimeParseException e) { - Map errorResponse = new HashMap<>(); - errorResponse.put("error", "Invalid date format. Please use yyyy-MM-dd."); - return ResponseEntity.badRequest().body(errorResponse); - } - } - } - - // Department filtering + // Validate and apply processed date range filter + if (processedStartDate != null || processedEndDate != null) { + if (processedStartDate == null || processedEndDate == null) { + Map errorResponse = new HashMap<>(); + errorResponse.put("error", "Both processedStartDate and processedEndDate are required."); + return ResponseEntity.badRequest().body(errorResponse); + } else { try { - if (institution != null && !institution.isEmpty()) { - criteria = applyDepartmentFilter(criteria, institution); - } - } catch (IllegalArgumentException e) { + var processedStart = LocalDate.parse(processedStartDate, dateFormat); + var processedEnd = LocalDate.parse(processedEndDate, dateFormat); + if (processedEnd.isBefore(processedStart)) { Map errorResponse = new HashMap<>(); - errorResponse.put("error", e.getMessage()); - return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(errorResponse); - } - - // URL filtering - if (url != null && !url.isEmpty()) { - criteria.and("url").regex(url, "i"); + errorResponse.put( + "error", "processedEndDate must be greater than or equal to processedStartDate."); + return ResponseEntity.badRequest().body(errorResponse); + } + criteria.and("processedDate").gte(processedStartDate).lte(processedEndDate); + } catch (DateTimeParseException e) { + Map errorResponse = new HashMap<>(); + errorResponse.put("error", "Invalid date format. Please use yyyy-MM-dd."); + return ResponseEntity.badRequest().body(errorResponse); } - - var query = new Query(criteria); - query - .fields() - .exclude("_id") - .exclude("section") - .exclude("oppositeLang") - .exclude("processed") - .exclude("contact") - .exclude("urlEntries") - .exclude("resolutionDate") - .exclude("resolution") - .exclude("topic") - .exclude("title") - .exclude("problem") - .exclude("dataOrigin") - .exclude("airTableSync") - .exclude("tags") - .exclude("personalInfoProcessed") - .exclude("autoTagProcessed") - .exclude("_class"); - - var documents = mongoTemplate.find(query, Document.class, "problem"); - return ResponseEntity.ok(documents); + } } - private Criteria applyDepartmentFilter(Criteria criteria, String department) { - var matchingVariations = new HashSet(); - for (Map.Entry> entry : institutionMappings.entrySet()) { - if (entry.getValue().stream().anyMatch(variation -> variation.equalsIgnoreCase(department))) { - matchingVariations.addAll(entry.getValue()); - } - } - - if (matchingVariations.isEmpty()) { - throw new IllegalArgumentException("Couldn't find department name: " + department); - } - - criteria.and("institution").in(matchingVariations); - return criteria; + // Department filtering + try { + if (institution != null && !institution.isEmpty()) { + criteria = applyDepartmentFilter(criteria, institution); + } + } catch (IllegalArgumentException e) { + Map errorResponse = new HashMap<>(); + errorResponse.put("error", e.getMessage()); + return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(errorResponse); } - @GetMapping("/exportExcel") - public void exportExcel(HttpServletRequest request, HttpServletResponse response) - throws IOException { - response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); - response.setHeader("Content-Disposition", "attachment; filename=\"feedback_export.xlsx\""); - - String[] titles = request.getParameterValues("titles[]"); - String language = request.getParameter("language"); - String department = request.getParameter("department"); - String comments = request.getParameter("comments"); - String theme = request.getParameter("theme"); - String section = request.getParameter("section"); - String url = request.getParameter("url"); - String startDate = request.getParameter("startDate"); - String endDate = request.getParameter("endDate"); - Boolean error_keyword = "true".equals(request.getParameter("error_keyword")); - - Criteria criteria = Criteria.where("processed").is("true"); - - // Apply filters (similar to the existing method) - var formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - if (startDate != null && endDate != null) { - var start = LocalDate.parse(startDate, formatter); - var end = LocalDate.parse(endDate, formatter); - criteria.and("problemDate").gte(start.format(formatter)).lte(end.format(formatter)); - } - if (theme != null && !theme.isEmpty()) { - criteria.and("theme").is(theme); - } - if (section != null && !section.isEmpty()) { - criteria.and("section").in(sectionMappings.getOrDefault(section.toLowerCase(), Collections.singletonList(section))); - } - if (language != null && !language.isEmpty()) { - criteria.and("language").is(language); - } - if (url != null && !url.isEmpty()) { - criteria.and("url").regex(url, "i"); - } - if (department != null && !department.isEmpty()) { - var matchingVariations = new HashSet(); - for (Map.Entry> entry : institutionMappings.entrySet()) { - if (entry.getValue().stream() - .anyMatch(variation -> variation.equalsIgnoreCase(department))) { - matchingVariations.addAll(entry.getValue()); - } - } - if (!matchingVariations.isEmpty()) { - criteria.and("institution").in(matchingVariations); - } - } - if (titles != null && titles.length > 0) { - var titleCriterias = new ArrayList(); - for (String title : titles) { - titleCriterias.add(Criteria.where("title").is(title)); - } - criteria = new Criteria().andOperator( - criteria, - new Criteria().orOperator(titleCriterias.toArray(new Criteria[0])) - ); - } - var regexCriteria = new ArrayList(); - - String trimmedComments = comments != null ? comments.trim() : null; - if (trimmedComments != null && !trimmedComments.isEmpty() && !"null".equalsIgnoreCase(trimmedComments)) { - String safeComments = escapeSpecialRegexCharacters(trimmedComments); - regexCriteria.add(Criteria.where("problemDetails").regex(safeComments, "i")); - } + // URL filtering + if (url != null && !url.isEmpty()) { + criteria.and("url").regex(url, "i"); + } - if (error_keyword) { - var keywords = new HashSet(); - keywords.addAll(errorKeywordService.getEnglishKeywords()); - keywords.addAll(errorKeywordService.getFrenchKeywords()); - keywords.addAll(errorKeywordService.getBilingualKeywords()); + var query = new Query(criteria); + query + .fields() + .exclude("_id") + .exclude("section") + .exclude("oppositeLang") + .exclude("processed") + .exclude("contact") + .exclude("urlEntries") + .exclude("resolutionDate") + .exclude("resolution") + .exclude("topic") + .exclude("title") + .exclude("problem") + .exclude("dataOrigin") + .exclude("airTableSync") + .exclude("tags") + .exclude("personalInfoProcessed") + .exclude("autoTagProcessed") + .exclude("_class"); + + var documents = mongoTemplate.find(query, Document.class, "problem"); + return ResponseEntity.ok(documents); + } + + private Criteria applyDepartmentFilter(Criteria criteria, String department) { + var matchingVariations = new HashSet(); + for (Map.Entry> entry : institutionMappings.entrySet()) { + if (entry.getValue().stream().anyMatch(variation -> variation.equalsIgnoreCase(department))) { + matchingVariations.addAll(entry.getValue()); + } + } - if (!keywords.isEmpty()) { - String combinedRegex = keywords.stream().map(Pattern::quote).collect(Collectors.joining("|")); - regexCriteria.add(Criteria.where("problemDetails").regex(combinedRegex, "i")); - } - } + if (matchingVariations.isEmpty()) { + throw new IllegalArgumentException("Couldn't find department name: " + department); + } - if (!regexCriteria.isEmpty()) { - criteria = new Criteria().andOperator(criteria, new Criteria().andOperator(regexCriteria.toArray(new Criteria[0]))); + criteria.and("institution").in(matchingVariations); + return criteria; + } + + @GetMapping("/exportExcel") + public void exportExcel(HttpServletRequest request, HttpServletResponse response) + throws IOException { + response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); + response.setHeader("Content-Disposition", "attachment; filename=\"feedback_export.xlsx\""); + + String[] titles = request.getParameterValues("titles[]"); + String language = request.getParameter("language"); + String department = request.getParameter("department"); + String comments = request.getParameter("comments"); + String theme = request.getParameter("theme"); + String section = request.getParameter("section"); + String url = request.getParameter("url"); + String startDate = request.getParameter("startDate"); + String endDate = request.getParameter("endDate"); + Boolean error_keyword = "true".equals(request.getParameter("error_keyword")); + + Criteria criteria = Criteria.where("processed").is("true"); + + // Apply filters (similar to the existing method) + var formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + if (startDate != null && endDate != null) { + var start = LocalDate.parse(startDate, formatter); + var end = LocalDate.parse(endDate, formatter); + criteria.and("problemDate").gte(start.format(formatter)).lte(end.format(formatter)); + } + if (theme != null && !theme.isEmpty()) { + criteria.and("theme").is(theme); + } + if (section != null && !section.isEmpty()) { + criteria + .and("section") + .in( + sectionMappings.getOrDefault( + section.toLowerCase(), Collections.singletonList(section))); + } + if (language != null && !language.isEmpty()) { + criteria.and("language").is(language); + } + if (url != null && !url.isEmpty()) { + criteria.and("url").regex(url, "i"); + } + if (department != null && !department.isEmpty()) { + var matchingVariations = new HashSet(); + for (Map.Entry> entry : institutionMappings.entrySet()) { + if (entry.getValue().stream() + .anyMatch(variation -> variation.equalsIgnoreCase(department))) { + matchingVariations.addAll(entry.getValue()); } + } + if (!matchingVariations.isEmpty()) { + criteria.and("institution").in(matchingVariations); + } + } + if (titles != null && titles.length > 0) { + var titleCriterias = new ArrayList(); + for (String title : titles) { + titleCriterias.add(Criteria.where("title").is(title)); + } + criteria = + new Criteria() + .andOperator( + criteria, new Criteria().orOperator(titleCriterias.toArray(new Criteria[0]))); + } + var regexCriteria = new ArrayList(); + + String trimmedComments = comments != null ? comments.trim() : null; + if (trimmedComments != null + && !trimmedComments.isEmpty() + && !"null".equalsIgnoreCase(trimmedComments)) { + String safeComments = escapeSpecialRegexCharacters(trimmedComments); + regexCriteria.add(Criteria.where("problemDetails").regex(safeComments, "i")); + } - var query = new Query(criteria); - query - .fields() - .include("problemDate") - .include("timeStamp") - .include("problemDetails") - .include("language") - .include("title") - .include("url") - .include("institution") - .include("section") - .include("theme") - .include("deviceType") - .include("browser"); - - // Use SXSSFWorkbook for better performance with large data - try (SXSSFWorkbook workbook = - new SXSSFWorkbook(100); // The argument (100) flushes rows after 100 are written - ServletOutputStream outputStream = response.getOutputStream()) { - - Sheet sheet = workbook.createSheet("Feedback Data"); - - // Create header row - String[] columns = { - "Problem Date", - "Time Stamp (UTC)", - "Problem Details", - "Language", - "Title", - "URL", - "Institution", - "Section", - "Theme", - "Device Type", - "Browser" - }; - var headerRow = sheet.createRow(0); - for (int i = 0; i < columns.length; i++) { - headerRow.createCell(i).setCellValue(columns[i]); - } - - // Stream and write data in batches - final int[] rowNum = {1}; - try (java.util.stream.Stream stream = mongoTemplate.stream(query, Problem.class)) { - stream.forEach( - problem -> { - Row row = sheet.createRow(rowNum[0]++); - row.createCell(0).setCellValue(problem.getProblemDate()); - row.createCell(1).setCellValue(problem.getTimeStamp()); - row.createCell(2).setCellValue(problem.getProblemDetails()); - row.createCell(3).setCellValue(problem.getLanguage()); - row.createCell(4).setCellValue(problem.getTitle()); - row.createCell(5).setCellValue(problem.getUrl()); - row.createCell(6).setCellValue(problem.getInstitution()); - row.createCell(7).setCellValue(problem.getSection()); - row.createCell(8).setCellValue(problem.getTheme()); - row.createCell(9).setCellValue(problem.getDeviceType()); - row.createCell(10).setCellValue(problem.getBrowser()); - - if (rowNum[0] % 100 == 0) { - try { - ((SXSSFSheet) sheet).flushRows(100); - } catch (IOException e) { - LOG.error("Error flushing rows", e); - } - } - }); - } - - // Write the workbook to the output stream - workbook.write(outputStream); - } catch (Exception e) { - LOG.error("Error exporting Excel", e); - response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - response.getWriter().write("Error exporting data: " + e.getMessage()); - } + if (error_keyword) { + var keywords = new HashSet(); + keywords.addAll(errorKeywordService.getEnglishKeywords()); + keywords.addAll(errorKeywordService.getFrenchKeywords()); + keywords.addAll(errorKeywordService.getBilingualKeywords()); + + if (!keywords.isEmpty()) { + String combinedRegex = + keywords.stream().map(Pattern::quote).collect(Collectors.joining("|")); + regexCriteria.add(Criteria.where("problemDetails").regex(combinedRegex, "i")); + } } - @GetMapping("/exportCSV") - public void exportCSV(HttpServletRequest request, HttpServletResponse response) - throws IOException { - response.setCharacterEncoding("UTF-8"); - response.setContentType("text/csv; charset=UTF-8"); - response.setHeader("Content-Disposition", "attachment; filename*=UTF-8''feedback_export.csv"); - - String[] titles = request.getParameterValues("titles[]"); - String language = request.getParameter("language"); - String department = request.getParameter("department"); - String comments = request.getParameter("comments"); - String theme = request.getParameter("theme"); - String section = request.getParameter("section"); - String url = request.getParameter("url"); - String startDate = request.getParameter("startDate"); - String endDate = request.getParameter("endDate"); - Boolean error_keyword = "true".equals(request.getParameter("error_keyword")); - - Criteria criteria = Criteria.where("processed").is("true"); - - // Apply filters (similar to the list method) - var formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - if (startDate != null && endDate != null) { - var start = LocalDate.parse(startDate, formatter); - var end = LocalDate.parse(endDate, formatter); - criteria.and("problemDate").gte(start.format(formatter)).lte(end.format(formatter)); - } + if (!regexCriteria.isEmpty()) { + criteria = + new Criteria() + .andOperator( + criteria, new Criteria().andOperator(regexCriteria.toArray(new Criteria[0]))); + } - if (theme != null && !theme.isEmpty()) { - criteria.and("theme").is(theme); - } - if (section != null && !section.isEmpty()) { - criteria.and("section").in(sectionMappings.getOrDefault(section.toLowerCase(), Collections.singletonList(section))); - } - if (language != null && !language.isEmpty()) { - criteria - .and("language") - .regex(Pattern.compile(Pattern.quote(language), Pattern.CASE_INSENSITIVE)); - } - if (url != null && !url.isEmpty()) { - criteria.and("url").regex(url, "i"); - } - if (department != null && !department.isEmpty()) { - var matchingVariations = new HashSet(); - for (Map.Entry> entry : institutionMappings.entrySet()) { - if (entry.getValue().stream() - .anyMatch(variation -> variation.equalsIgnoreCase(department))) { - matchingVariations.addAll(entry.getValue()); + var query = new Query(criteria); + query + .fields() + .include("problemDate") + .include("timeStamp") + .include("problemDetails") + .include("language") + .include("title") + .include("url") + .include("institution") + .include("section") + .include("theme") + .include("deviceType") + .include("browser"); + + // Use SXSSFWorkbook for better performance with large data + try (SXSSFWorkbook workbook = + new SXSSFWorkbook(100); // The argument (100) flushes rows after 100 are written + ServletOutputStream outputStream = response.getOutputStream()) { + + Sheet sheet = workbook.createSheet("Feedback Data"); + + // Create header row + String[] columns = { + "Problem Date", + "Time Stamp (UTC)", + "Problem Details", + "Language", + "Title", + "URL", + "Institution", + "Section", + "Theme", + "Device Type", + "Browser" + }; + var headerRow = sheet.createRow(0); + for (int i = 0; i < columns.length; i++) { + headerRow.createCell(i).setCellValue(columns[i]); + } + + // Stream and write data in batches + final int[] rowNum = {1}; + try (java.util.stream.Stream stream = mongoTemplate.stream(query, Problem.class)) { + stream.forEach( + problem -> { + Row row = sheet.createRow(rowNum[0]++); + row.createCell(0).setCellValue(problem.getProblemDate()); + row.createCell(1).setCellValue(problem.getTimeStamp()); + row.createCell(2).setCellValue(problem.getProblemDetails()); + row.createCell(3).setCellValue(problem.getLanguage()); + row.createCell(4).setCellValue(problem.getTitle()); + row.createCell(5).setCellValue(problem.getUrl()); + row.createCell(6).setCellValue(problem.getInstitution()); + row.createCell(7).setCellValue(problem.getSection()); + row.createCell(8).setCellValue(problem.getTheme()); + row.createCell(9).setCellValue(problem.getDeviceType()); + row.createCell(10).setCellValue(problem.getBrowser()); + + if (rowNum[0] % 100 == 0) { + try { + ((SXSSFSheet) sheet).flushRows(100); + } catch (IOException e) { + LOG.error("Error flushing rows", e); } - } - if (!matchingVariations.isEmpty()) { - criteria.and("institution").in(matchingVariations); - } - } - if (titles != null && titles.length > 0) { - var titleCriterias = new ArrayList(); - for (String title : titles) { - titleCriterias.add(Criteria.where("title").is(title)); - } - criteria = new Criteria().andOperator( - criteria, - new Criteria().orOperator(titleCriterias.toArray(new Criteria[0])) - ); - } - var regexCriteria = new ArrayList(); + } + }); + } + + // Write the workbook to the output stream + workbook.write(outputStream); + } catch (Exception e) { + LOG.error("Error exporting Excel", e); + response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + response.getWriter().write("Error exporting data: " + e.getMessage()); + } + } + + @GetMapping("/exportCSV") + public void exportCSV(HttpServletRequest request, HttpServletResponse response) + throws IOException { + response.setCharacterEncoding("UTF-8"); + response.setContentType("text/csv; charset=UTF-8"); + response.setHeader("Content-Disposition", "attachment; filename*=UTF-8''feedback_export.csv"); + + String[] titles = request.getParameterValues("titles[]"); + String language = request.getParameter("language"); + String department = request.getParameter("department"); + String comments = request.getParameter("comments"); + String theme = request.getParameter("theme"); + String section = request.getParameter("section"); + String url = request.getParameter("url"); + String startDate = request.getParameter("startDate"); + String endDate = request.getParameter("endDate"); + Boolean error_keyword = "true".equals(request.getParameter("error_keyword")); + + Criteria criteria = Criteria.where("processed").is("true"); + + // Apply filters (similar to the list method) + var formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + if (startDate != null && endDate != null) { + var start = LocalDate.parse(startDate, formatter); + var end = LocalDate.parse(endDate, formatter); + criteria.and("problemDate").gte(start.format(formatter)).lte(end.format(formatter)); + } - if (comments != null && !comments.trim().isEmpty() && !"null".equalsIgnoreCase(comments.trim())) { - String safeComments = escapeSpecialRegexCharacters(comments.trim()); - regexCriteria.add(Criteria.where("problemDetails").regex(safeComments, "i")); + if (theme != null && !theme.isEmpty()) { + criteria.and("theme").is(theme); + } + if (section != null && !section.isEmpty()) { + criteria + .and("section") + .in( + sectionMappings.getOrDefault( + section.toLowerCase(), Collections.singletonList(section))); + } + if (language != null && !language.isEmpty()) { + criteria + .and("language") + .regex(Pattern.compile(Pattern.quote(language), Pattern.CASE_INSENSITIVE)); + } + if (url != null && !url.isEmpty()) { + criteria.and("url").regex(url, "i"); + } + if (department != null && !department.isEmpty()) { + var matchingVariations = new HashSet(); + for (Map.Entry> entry : institutionMappings.entrySet()) { + if (entry.getValue().stream() + .anyMatch(variation -> variation.equalsIgnoreCase(department))) { + matchingVariations.addAll(entry.getValue()); } + } + if (!matchingVariations.isEmpty()) { + criteria.and("institution").in(matchingVariations); + } + } + if (titles != null && titles.length > 0) { + var titleCriterias = new ArrayList(); + for (String title : titles) { + titleCriterias.add(Criteria.where("title").is(title)); + } + criteria = + new Criteria() + .andOperator( + criteria, new Criteria().orOperator(titleCriterias.toArray(new Criteria[0]))); + } + var regexCriteria = new ArrayList(); - if (error_keyword) { - var keywords = new HashSet(); - keywords.addAll(errorKeywordService.getEnglishKeywords()); - keywords.addAll(errorKeywordService.getFrenchKeywords()); - keywords.addAll(errorKeywordService.getBilingualKeywords()); + if (comments != null + && !comments.trim().isEmpty() + && !"null".equalsIgnoreCase(comments.trim())) { + String safeComments = escapeSpecialRegexCharacters(comments.trim()); + regexCriteria.add(Criteria.where("problemDetails").regex(safeComments, "i")); + } - if (!keywords.isEmpty()) { - String combinedRegex = keywords.stream().map(Pattern::quote).collect(Collectors.joining("|")); - regexCriteria.add(Criteria.where("problemDetails").regex(combinedRegex, "i")); - } - } + if (error_keyword) { + var keywords = new HashSet(); + keywords.addAll(errorKeywordService.getEnglishKeywords()); + keywords.addAll(errorKeywordService.getFrenchKeywords()); + keywords.addAll(errorKeywordService.getBilingualKeywords()); + + if (!keywords.isEmpty()) { + String combinedRegex = + keywords.stream().map(Pattern::quote).collect(Collectors.joining("|")); + regexCriteria.add(Criteria.where("problemDetails").regex(combinedRegex, "i")); + } + } - if (!regexCriteria.isEmpty()) { - criteria = new Criteria().andOperator(criteria, new Criteria().andOperator(regexCriteria.toArray(new Criteria[0]))); - } + if (!regexCriteria.isEmpty()) { + criteria = + new Criteria() + .andOperator( + criteria, new Criteria().andOperator(regexCriteria.toArray(new Criteria[0]))); + } - var query = new Query(criteria); - query - .fields() - .include("problemDate") - .include("timeStamp") - .include("problemDetails") - .include("language") - .include("title") - .include("url") - .include("institution") - .include("section") - .include("theme") - .include("deviceType") - .include("browser"); - - - // Stream results directly to the response - try (Writer writer = response.getWriter()) { - writer.write("\uFEFF"); - - // Write CSV header - writer.write(""" + var query = new Query(criteria); + query + .fields() + .include("problemDate") + .include("timeStamp") + .include("problemDetails") + .include("language") + .include("title") + .include("url") + .include("institution") + .include("section") + .include("theme") + .include("deviceType") + .include("browser"); + + // Stream results directly to the response + try (Writer writer = response.getWriter()) { + writer.write("\uFEFF"); + + // Write CSV header + writer.write( + """ Problem Date,Time Stamp (UTC),Problem Details,Language,Title,URL,Institution,Section,Theme,Device Type,Browser """); - // Stream and write data - try (java.util.stream.Stream stream = mongoTemplate.stream(query, Problem.class)) { - stream.forEach(problem -> { - try { - writer.write( - String.format( - "%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n", - escapeCSV(problem.getProblemDate()), - escapeCSV(problem.getTimeStamp()), - escapeCSV(problem.getProblemDetails()), - escapeCSV(problem.getLanguage()), - escapeCSV(problem.getTitle()), - escapeCSV(problem.getUrl()), - escapeCSV(problem.getInstitution()), - escapeCSV(problem.getSection()), - escapeCSV(problem.getTheme()), - escapeCSV(problem.getDeviceType()), - escapeCSV(problem.getBrowser()) - )); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - } - } + // Stream and write data + try (java.util.stream.Stream stream = mongoTemplate.stream(query, Problem.class)) { + stream.forEach( + problem -> { + try { + writer.write( + String.format( + "%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n", + escapeCSV(problem.getProblemDate()), + escapeCSV(problem.getTimeStamp()), + escapeCSV(problem.getProblemDetails()), + escapeCSV(problem.getLanguage()), + escapeCSV(problem.getTitle()), + escapeCSV(problem.getUrl()), + escapeCSV(problem.getInstitution()), + escapeCSV(problem.getSection()), + escapeCSV(problem.getTheme()), + escapeCSV(problem.getDeviceType()), + escapeCSV(problem.getBrowser()))); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } } + } - private String escapeCSV(String value) { - if (value == null) { - return ""; - } - return "\"" + value.replace("\"", "\"\"") + "\""; + private String escapeCSV(String value) { + if (value == null) { + return ""; } + return "\"" + value.replace("\"", "\"\"") + "\""; + } + + @GetMapping(value = "/pageFeedback") + public ModelAndView pageFeedback(HttpServletRequest request) throws Exception { + var mav = new ModelAndView(); + String lang = (String) request.getSession().getAttribute("lang"); + + // Fetch the aggregation results + var dateMap = problemDateService.getProblemDates(); + + if (dateMap != null) { + mav.addObject("earliestDate", dateMap.get("earliestDate")); + mav.addObject("latestDate", dateMap.get("latestDate")); + } else { + // Handle the case where no dates are returned + mav.addObject("earliestDate", "N/A"); + mav.addObject("latestDate", "N/A"); + } + mav.addObject("lang", lang); - @GetMapping(value = "/pageFeedback") - public ModelAndView pageFeedback(HttpServletRequest request) throws Exception { - var mav = new ModelAndView(); - String lang = (String) request.getSession().getAttribute("lang"); - - // Fetch the aggregation results - var dateMap = problemDateService.getProblemDates(); - - if (dateMap != null) { - mav.addObject("earliestDate", dateMap.get("earliestDate")); - mav.addObject("latestDate", dateMap.get("latestDate")); - } else { - // Handle the case where no dates are returned - mav.addObject("earliestDate", "N/A"); - mav.addObject("latestDate", "N/A"); - } - mav.addObject("lang", lang); + mav.setViewName("pageFeedback_" + lang); + return mav; + } - mav.setViewName("pageFeedback_" + lang); - return mav; + private boolean containsErrorKeywords(Problem problem) { + if (problem == null || problem.getProblemDetails() == null) { + return false; } - - private boolean containsErrorKeywords(Problem problem) { - if (problem == null || problem.getProblemDetails() == null) { - return false; - } - return errorKeywordService.containsErrorKeywords( - problem.getProblemDetails(), problem.getLanguage()); + return errorKeywordService.containsErrorKeywords( + problem.getProblemDetails(), problem.getLanguage()); + } + + @GetMapping(value = "/feedbackData") + @ResponseBody + public DataTablesOutput list(@Valid DataTablesInput input, HttpServletRequest request) { + String pageLang = (String) request.getSession().getAttribute("lang"); + String language = request.getParameter("language"); + String department = request.getParameter("department"); + String comments = request.getParameter("comments"); + String theme = request.getParameter("theme"); + String section = request.getParameter("section"); + String url = request.getParameter("url"); + Boolean error_keyword = "true".equals(request.getParameter("error_keyword")); + String startDate = request.getParameter("startDate"); + String endDate = request.getParameter("endDate"); + String[] titles = request.getParameterValues("titles[]"); + + var criteria = Criteria.where("processed").is("true"); + + var formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + if (startDate != null && endDate != null) { + var start = LocalDate.parse(startDate, formatter); + var end = LocalDate.parse(endDate, formatter); + criteria.and("problemDate").gte(start.format(formatter)).lte(end.format(formatter)); } - @GetMapping(value = "/feedbackData") - @ResponseBody - public DataTablesOutput list(@Valid DataTablesInput input, HttpServletRequest request) { - String pageLang = (String) request.getSession().getAttribute("lang"); - String language = request.getParameter("language"); - String department = request.getParameter("department"); - String comments = request.getParameter("comments"); - String theme = request.getParameter("theme"); - String section = request.getParameter("section"); - String url = request.getParameter("url"); - Boolean error_keyword = "true".equals(request.getParameter("error_keyword")); - String startDate = request.getParameter("startDate"); - String endDate = request.getParameter("endDate"); - String[] titles = request.getParameterValues("titles[]"); - - var criteria = Criteria.where("processed").is("true"); - - var formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - if (startDate != null && endDate != null) { - var start = LocalDate.parse(startDate, formatter); - var end = LocalDate.parse(endDate, formatter); - criteria.and("problemDate").gte(start.format(formatter)).lte(end.format(formatter)); - } - - if (theme != null && !theme.isEmpty()) { - criteria.and("theme").is(theme); - } - if (section != null && !section.isEmpty()) { - criteria.and("section").in(sectionMappings.getOrDefault(section.toLowerCase(), Collections.singletonList(section))); - } - // Language filtering (existing logic) - if (language != null && !language.isEmpty()) { - criteria.and("language").is(language); - } - - // URL filtering - if (url != null && !url.isEmpty()) { - criteria.and("url").regex(url, "i"); // 'i' for case-insensitive matching - } - // Department filtering based on institutionMappings - if (department != null && !department.isEmpty()) { - var matchingVariations = new HashSet(); - // Filter variations based on department: - for (Map.Entry> entry : institutionMappings.entrySet()) { - if (entry.getValue().stream() - .anyMatch(variation -> variation.equalsIgnoreCase(department))) { - matchingVariations.addAll(entry.getValue()); - } - } - if (!matchingVariations.isEmpty()) { - criteria.and("institution").in(matchingVariations); - } - } - if (titles != null && titles.length > 0) { - // Create a list to hold the title criteria - var titleCriterias = new ArrayList(); - // Iterate over the titles and add each one as a criterion - for (String title : titles) { - titleCriterias.add(Criteria.where("title").is(title)); - } - // Combine all title criteria using AND operation - criteria = new Criteria().andOperator( - criteria, - new Criteria().orOperator(titleCriterias.toArray(new Criteria[0])) - ); - System.out.println("Titles received: " + Arrays.toString(titles)); - } - - var regexCriteria = new ArrayList(); - // Comments filtering - if (comments != null && !comments.trim().isEmpty() - && !"null".equalsIgnoreCase(comments.trim())) { - String safeComments = escapeSpecialRegexCharacters(comments.trim()); - regexCriteria.add(Criteria.where("problemDetails").regex(safeComments, "i")); - } - //error keywords filtering - if (error_keyword) { - var keywords = new HashSet(); - keywords.addAll(errorKeywordService.getEnglishKeywords()); - keywords.addAll(errorKeywordService.getFrenchKeywords()); - keywords.addAll(errorKeywordService.getBilingualKeywords()); - - if (!keywords.isEmpty()) { - String combinedRegex = keywords.stream().map(Pattern::quote).collect(Collectors.joining("|")); - regexCriteria.add(Criteria.where("problemDetails").regex(combinedRegex, "i")); - } - } + if (theme != null && !theme.isEmpty()) { + criteria.and("theme").is(theme); + } + if (section != null && !section.isEmpty()) { + criteria + .and("section") + .in( + sectionMappings.getOrDefault( + section.toLowerCase(), Collections.singletonList(section))); + } + // Language filtering (existing logic) + if (language != null && !language.isEmpty()) { + criteria.and("language").is(language); + } - if (!regexCriteria.isEmpty()) { - criteria = new Criteria().andOperator(criteria, new Criteria().andOperator(regexCriteria.toArray(new Criteria[0]))); + // URL filtering + if (url != null && !url.isEmpty()) { + criteria.and("url").regex(url, "i"); // 'i' for case-insensitive matching + } + // Department filtering based on institutionMappings + if (department != null && !department.isEmpty()) { + var matchingVariations = new HashSet(); + // Filter variations based on department: + for (Map.Entry> entry : institutionMappings.entrySet()) { + if (entry.getValue().stream() + .anyMatch(variation -> variation.equalsIgnoreCase(department))) { + matchingVariations.addAll(entry.getValue()); } + } + if (!matchingVariations.isEmpty()) { + criteria.and("institution").in(matchingVariations); + } + } + if (titles != null && titles.length > 0) { + // Create a list to hold the title criteria + var titleCriterias = new ArrayList(); + // Iterate over the titles and add each one as a criterion + for (String title : titles) { + titleCriterias.add(Criteria.where("title").is(title)); + } + // Combine all title criteria using AND operation + criteria = + new Criteria() + .andOperator( + criteria, new Criteria().orOperator(titleCriterias.toArray(new Criteria[0]))); + System.out.println("Titles received: " + Arrays.toString(titles)); + } - DataTablesOutput results; - - - // Use the cached total count when no filters narrow the result set, - // to avoid an expensive count query against CosmosDB. - boolean isFiltered = (startDate != null && endDate != null) - || (language != null && !language.isEmpty()) - || (department != null && !department.isEmpty()) - || (theme != null && !theme.isEmpty()) - || (section != null && !section.isEmpty()) - || (url != null && !url.isEmpty()) - || (comments != null && !comments.isEmpty()) - || (titles != null && titles.length > 0) - || error_keyword; - long cachedCount = isFiltered ? -1 : problemCacheService.getProcessedProblems().size(); - results = problemRepository.findAll(input, criteria, cachedCount); - - // Update institution names in the results based on the language - setInstitution(results, pageLang); - // Return the updated results - return results; + var regexCriteria = new ArrayList(); + // Comments filtering + if (comments != null + && !comments.trim().isEmpty() + && !"null".equalsIgnoreCase(comments.trim())) { + String safeComments = escapeSpecialRegexCharacters(comments.trim()); + regexCriteria.add(Criteria.where("problemDetails").regex(safeComments, "i")); + } + // error keywords filtering + if (error_keyword) { + var keywords = new HashSet(); + keywords.addAll(errorKeywordService.getEnglishKeywords()); + keywords.addAll(errorKeywordService.getFrenchKeywords()); + keywords.addAll(errorKeywordService.getBilingualKeywords()); + + if (!keywords.isEmpty()) { + String combinedRegex = + keywords.stream().map(Pattern::quote).collect(Collectors.joining("|")); + regexCriteria.add(Criteria.where("problemDetails").regex(combinedRegex, "i")); + } } - /** - * Escapes special regex characters in the input string. - * - * @param input The string to escape. - * @return A string with special regex characters escaped. - */ - private String escapeSpecialRegexCharacters(String input) { - // Escape all regex metacharacters - return input.replaceAll("([\\\\.^$|()\\[\\]{}*+?])", "\\\\$1"); + if (!regexCriteria.isEmpty()) { + criteria = + new Criteria() + .andOperator( + criteria, new Criteria().andOperator(regexCriteria.toArray(new Criteria[0]))); } - private void setInstitution(DataTablesOutput problems, String lang) { - for (Problem problem : problems.getData()) { - String currentInstitution = problem.getInstitution(); - for (Map.Entry> entry : institutionMappings.entrySet()) { - if (entry.getValue().contains(currentInstitution)) { - // Assuming the translated institution name is at index 1 for French and index 0 - // for other languages - problem.setInstitution(entry.getValue().get(lang.equalsIgnoreCase("fr") ? 1 : 0)); - break; // Exit the loop once the institution is found and updated - } - } + DataTablesOutput results; + + // Use the cached total count when no filters narrow the result set, + // to avoid an expensive count query against CosmosDB. + boolean isFiltered = + (startDate != null && endDate != null) + || (language != null && !language.isEmpty()) + || (department != null && !department.isEmpty()) + || (theme != null && !theme.isEmpty()) + || (section != null && !section.isEmpty()) + || (url != null && !url.isEmpty()) + || (comments != null && !comments.isEmpty()) + || (titles != null && titles.length > 0) + || error_keyword; + long cachedCount = isFiltered ? -1 : problemCacheService.getProcessedProblems().size(); + results = problemRepository.findAll(input, criteria, cachedCount); + + // Update institution names in the results based on the language + setInstitution(results, pageLang); + // Return the updated results + return results; + } + + /** + * Escapes special regex characters in the input string. + * + * @param input The string to escape. + * @return A string with special regex characters escaped. + */ + private String escapeSpecialRegexCharacters(String input) { + // Escape all regex metacharacters + return input.replaceAll("([\\\\.^$|()\\[\\]{}*+?])", "\\\\$1"); + } + + private void setInstitution(DataTablesOutput problems, String lang) { + for (Problem problem : problems.getData()) { + String currentInstitution = problem.getInstitution(); + for (Map.Entry> entry : institutionMappings.entrySet()) { + if (entry.getValue().contains(currentInstitution)) { + // Assuming the translated institution name is at index 1 for French and index 0 + // for other languages + problem.setInstitution(entry.getValue().get(lang.equalsIgnoreCase("fr") ? 1 : 0)); + break; // Exit the loop once the institution is found and updated } + } } - + } } diff --git a/src/main/java/ca/gc/tbs/controller/TopTaskController.java b/src/main/java/ca/gc/tbs/controller/TopTaskController.java index aed9f5fc..2cc322ba 100644 --- a/src/main/java/ca/gc/tbs/controller/TopTaskController.java +++ b/src/main/java/ca/gc/tbs/controller/TopTaskController.java @@ -1,5 +1,15 @@ package ca.gc.tbs.controller; +import ca.gc.tbs.domain.TopTaskSurvey; +import ca.gc.tbs.domain.User; +import ca.gc.tbs.repository.TopTaskRepository; +import ca.gc.tbs.security.JWTUtil; +import ca.gc.tbs.service.ProblemDateService; +import ca.gc.tbs.service.UserService; +import jakarta.servlet.ServletOutputStream; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import jakarta.validation.Valid; import java.io.IOException; import java.io.Writer; import java.time.LocalDate; @@ -14,12 +24,6 @@ import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; - -import jakarta.servlet.ServletOutputStream; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; -import jakarta.validation.Valid; - import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.xssf.streaming.SXSSFSheet; @@ -43,13 +47,6 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.servlet.ModelAndView; -import ca.gc.tbs.domain.TopTaskSurvey; -import ca.gc.tbs.domain.User; -import ca.gc.tbs.repository.TopTaskRepository; -import ca.gc.tbs.security.JWTUtil; -import ca.gc.tbs.service.ProblemDateService; -import ca.gc.tbs.service.UserService; - @Controller public class TopTaskController { @@ -81,6 +78,7 @@ public TopTaskController( this.mongoTemplate = mongoTemplate; this.jwtUtil = jwtUtil; } + private static final Map> institutionMappings = new HashMap<>(); static { @@ -504,7 +502,9 @@ public String totalTaskCount() { return String.valueOf(totalTaskCount); } - @RequestMapping(value = "/topTaskData", method = {RequestMethod.GET, RequestMethod.POST}) + @RequestMapping( + value = "/topTaskData", + method = {RequestMethod.GET, RequestMethod.POST}) @ResponseBody public DataTablesOutput list( @Valid DataTablesInput input, HttpServletRequest request) { @@ -513,7 +513,7 @@ public DataTablesOutput list( String departmentFilterVal = request.getParameter("department"); String themeFilterVal = request.getParameter("theme"); if (themeFilterVal != null) { - themeFilterVal = themeFilterVal.trim().replaceAll("\\s+", " "); + themeFilterVal = themeFilterVal.trim().replaceAll("\\s+", " "); } String[] taskFilterVals = request.getParameterValues("tasks[]"); String startDateVal = request.getParameter("startDate"); @@ -521,7 +521,8 @@ public DataTablesOutput list( String groupFilterVal = request.getParameter("group"); String language = request.getParameter("language"); String includeCommentsOnlyParam = request.getParameter("includeCommentsOnly"); - boolean includeCommentsOnly = includeCommentsOnlyParam != null && includeCommentsOnlyParam.equals("true"); + boolean includeCommentsOnly = + includeCommentsOnlyParam != null && includeCommentsOnlyParam.equals("true"); String taskCompletionFilterVal = request.getParameter("taskCompletion"); String comments = request.getParameter("comments"); @@ -572,49 +573,60 @@ public DataTablesOutput list( } else if (!combinedOrCriteria.isEmpty()) { criteria.orOperator(combinedOrCriteria.toArray(new Criteria[0])); } - //taskCompletion filter - if (taskCompletionFilterVal != null && !taskCompletionFilterVal.isEmpty()) { - List allowed = new ArrayList<>(); - if (taskCompletionFilterVal.equals("Yes")) { - allowed.add("Yes / Oui"); - } else if (taskCompletionFilterVal.equals("No")) { - allowed.add("No / Non"); - } else if (taskCompletionFilterVal.equals("I started this survey before I finished my visit")) { - allowed.add("I started this survey before I finished my visit / J’ai commencé ce sondage avant d’avoir terminé ma visite"); - } - if (!allowed.isEmpty()) { - criteria.and("taskCompletion").in(allowed); - } + // taskCompletion filter + if (taskCompletionFilterVal != null && !taskCompletionFilterVal.isEmpty()) { + List allowed = new ArrayList<>(); + if (taskCompletionFilterVal.equals("Yes")) { + allowed.add("Yes / Oui"); + } else if (taskCompletionFilterVal.equals("No")) { + allowed.add("No / Non"); + } else if (taskCompletionFilterVal.equals( + "I started this survey before I finished my visit")) { + allowed.add( + "I started this survey before I finished my visit / J’ai commencé ce sondage avant" + + " d’avoir terminé ma visite"); } - // Comments filtering - if (comments != null && !comments.trim().isEmpty() && !"null".equalsIgnoreCase(comments. trim())) { - String escapedComment = escapeSpecialRegexCharacters(comments.trim()); - List commentCriteria = new ArrayList<>(); - commentCriteria.add(Criteria.where("taskImproveComment").regex(escapedComment, "i")); - commentCriteria.add(Criteria.where("taskWhyNotComment").regex(escapedComment, "i")); - commentCriteria.add(Criteria.where("taskOther").regex(escapedComment, "i")); - - criteria = new Criteria().andOperator(criteria, new Criteria().orOperator(commentCriteria.toArray(new Criteria[0]))); + if (!allowed.isEmpty()) { + criteria.and("taskCompletion").in(allowed); } - + } + // Comments filtering + if (comments != null + && !comments.trim().isEmpty() + && !"null".equalsIgnoreCase(comments.trim())) { + String escapedComment = escapeSpecialRegexCharacters(comments.trim()); + List commentCriteria = new ArrayList<>(); + commentCriteria.add(Criteria.where("taskImproveComment").regex(escapedComment, "i")); + commentCriteria.add(Criteria.where("taskWhyNotComment").regex(escapedComment, "i")); + commentCriteria.add(Criteria.where("taskOther").regex(escapedComment, "i")); + + criteria = + new Criteria() + .andOperator( + criteria, new Criteria().orOperator(commentCriteria.toArray(new Criteria[0]))); + } List distinctTaskCounts = topTaskRepository.findDistinctTaskCountsWithFilters(criteria); totalDistinctTasks = distinctTaskCounts.size(); // Use estimatedDocumentCount (metadata-based, instant) when no filters are applied, // to avoid an expensive count query against CosmosDB. - boolean isFiltered = (startDateVal != null && endDateVal != null) - || (language != null && !language.isEmpty()) - || (departmentFilterVal != null && !departmentFilterVal.isEmpty()) - || (themeFilterVal != null && !themeFilterVal.isEmpty()) - || (groupFilterVal != null && !groupFilterVal.isEmpty()) - || (taskFilterVals != null && taskFilterVals.length > 0) - || (taskCompletionFilterVal != null && !taskCompletionFilterVal.isEmpty()) - || (comments != null && !comments.trim().isEmpty() && !"null".equalsIgnoreCase(comments.trim())) - || includeCommentsOnly; - long cachedCount = isFiltered ? -1 - : mongoTemplate.getCollection("toptasksurvey").estimatedDocumentCount(); - DataTablesOutput results = topTaskRepository.findAll(input, criteria, cachedCount); + boolean isFiltered = + (startDateVal != null && endDateVal != null) + || (language != null && !language.isEmpty()) + || (departmentFilterVal != null && !departmentFilterVal.isEmpty()) + || (themeFilterVal != null && !themeFilterVal.isEmpty()) + || (groupFilterVal != null && !groupFilterVal.isEmpty()) + || (taskFilterVals != null && taskFilterVals.length > 0) + || (taskCompletionFilterVal != null && !taskCompletionFilterVal.isEmpty()) + || (comments != null + && !comments.trim().isEmpty() + && !"null".equalsIgnoreCase(comments.trim())) + || includeCommentsOnly; + long cachedCount = + isFiltered ? -1 : mongoTemplate.getCollection("toptasksurvey").estimatedDocumentCount(); + DataTablesOutput results = + topTaskRepository.findAll(input, criteria, cachedCount); totalTaskCount = (int) results.getRecordsFiltered(); return results; @@ -692,47 +704,48 @@ public void exportTopTaskExcel(HttpServletRequest request, HttpServletResponse r final int[] rowNum = {1}; try (ServletOutputStream outputStream = response.getOutputStream(); - java.util.stream.Stream stream = mongoTemplate.stream(query, TopTaskSurvey.class)) { + java.util.stream.Stream stream = + mongoTemplate.stream(query, TopTaskSurvey.class)) { stream.forEach( - survey -> { - try { - Row row = sheet.createRow(rowNum[0]++); - row.createCell(0).setCellValue(survey.getDateTime()); - row.createCell(1).setCellValue(survey.getTimeStamp()); - row.createCell(2).setCellValue(survey.getSurveyReferrer()); - row.createCell(3).setCellValue(survey.getLanguage()); - row.createCell(4).setCellValue(survey.getDevice()); - row.createCell(5).setCellValue(survey.getScreener()); - row.createCell(6).setCellValue(survey.getDept()); - row.createCell(7).setCellValue(survey.getTheme()); - row.createCell(8).setCellValue(survey.getThemeOther()); - row.createCell(9).setCellValue(survey.getGrouping()); - row.createCell(10).setCellValue(survey.getTask()); - row.createCell(11).setCellValue(survey.getTaskOther()); - row.createCell(12).setCellValue(survey.getTaskSatisfaction()); - row.createCell(13).setCellValue(survey.getTaskEase()); - row.createCell(14).setCellValue(survey.getTaskCompletion()); - row.createCell(15).setCellValue(survey.getTaskImprove()); - row.createCell(16).setCellValue(survey.getTaskImproveComment()); - row.createCell(17).setCellValue(survey.getTaskWhyNot()); - row.createCell(18).setCellValue(survey.getTaskWhyNotComment()); - row.createCell(19).setCellValue(survey.getTaskSampling()); - row.createCell(20).setCellValue(survey.getSamplingInvitation()); - row.createCell(21).setCellValue(survey.getSamplingGC()); - row.createCell(22).setCellValue(survey.getSamplingCanada()); - row.createCell(23).setCellValue(survey.getSamplingTheme()); - row.createCell(24).setCellValue(survey.getSamplingInstitution()); - row.createCell(25).setCellValue(survey.getSamplingGrouping()); - row.createCell(26).setCellValue(survey.getSamplingTask()); - - if (rowNum[0] % 100 == 0) { - ((SXSSFSheet) sheet).flushRows(100); - LOG.debug("Flushed {} rows", rowNum[0]); - } - } catch (Exception e) { - LOG.error("Error writing row {}: {}", rowNum[0], e.getMessage()); - } - }); + survey -> { + try { + Row row = sheet.createRow(rowNum[0]++); + row.createCell(0).setCellValue(survey.getDateTime()); + row.createCell(1).setCellValue(survey.getTimeStamp()); + row.createCell(2).setCellValue(survey.getSurveyReferrer()); + row.createCell(3).setCellValue(survey.getLanguage()); + row.createCell(4).setCellValue(survey.getDevice()); + row.createCell(5).setCellValue(survey.getScreener()); + row.createCell(6).setCellValue(survey.getDept()); + row.createCell(7).setCellValue(survey.getTheme()); + row.createCell(8).setCellValue(survey.getThemeOther()); + row.createCell(9).setCellValue(survey.getGrouping()); + row.createCell(10).setCellValue(survey.getTask()); + row.createCell(11).setCellValue(survey.getTaskOther()); + row.createCell(12).setCellValue(survey.getTaskSatisfaction()); + row.createCell(13).setCellValue(survey.getTaskEase()); + row.createCell(14).setCellValue(survey.getTaskCompletion()); + row.createCell(15).setCellValue(survey.getTaskImprove()); + row.createCell(16).setCellValue(survey.getTaskImproveComment()); + row.createCell(17).setCellValue(survey.getTaskWhyNot()); + row.createCell(18).setCellValue(survey.getTaskWhyNotComment()); + row.createCell(19).setCellValue(survey.getTaskSampling()); + row.createCell(20).setCellValue(survey.getSamplingInvitation()); + row.createCell(21).setCellValue(survey.getSamplingGC()); + row.createCell(22).setCellValue(survey.getSamplingCanada()); + row.createCell(23).setCellValue(survey.getSamplingTheme()); + row.createCell(24).setCellValue(survey.getSamplingInstitution()); + row.createCell(25).setCellValue(survey.getSamplingGrouping()); + row.createCell(26).setCellValue(survey.getSamplingTask()); + + if (rowNum[0] % 100 == 0) { + ((SXSSFSheet) sheet).flushRows(100); + LOG.debug("Flushed {} rows", rowNum[0]); + } + } catch (Exception e) { + LOG.error("Error writing row {}: {}", rowNum[0], e.getMessage()); + } + }); LOG.info("Writing {} rows to Excel file", rowNum[0] - 1); workbook.write(outputStream); @@ -795,44 +808,45 @@ public void exportTopTaskCSV(HttpServletRequest request, HttpServletResponse res + " Theme,Sampling Institution,Sampling Grouping,Sampling Task\n"); // Stream and write data - try (java.util.stream.Stream stream = mongoTemplate.stream(query, TopTaskSurvey.class)) { + try (java.util.stream.Stream stream = + mongoTemplate.stream(query, TopTaskSurvey.class)) { stream.forEach( - survey -> { - try { - writer.write( - String.format( - "%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n", - escapeCSV(survey.getDateTime()), - escapeCSV(survey.getTimeStamp()), - escapeCSV(survey.getSurveyReferrer()), - escapeCSV(survey.getLanguage()), - escapeCSV(survey.getDevice()), - escapeCSV(survey.getScreener()), - escapeCSV(survey.getDept()), - escapeCSV(survey.getTheme()), - escapeCSV(survey.getThemeOther()), - escapeCSV(survey.getGrouping()), - escapeCSV(survey.getTask()), - escapeCSV(survey.getTaskOther()), - escapeCSV(survey.getTaskSatisfaction()), - escapeCSV(survey.getTaskEase()), - escapeCSV(survey.getTaskCompletion()), - escapeCSV(survey.getTaskImprove()), - escapeCSV(survey.getTaskImproveComment()), - escapeCSV(survey.getTaskWhyNot()), - escapeCSV(survey.getTaskWhyNotComment()), - escapeCSV(survey.getTaskSampling()), - escapeCSV(survey.getSamplingInvitation()), - escapeCSV(survey.getSamplingGC()), - escapeCSV(survey.getSamplingCanada()), - escapeCSV(survey.getSamplingTheme()), - escapeCSV(survey.getSamplingInstitution()), - escapeCSV(survey.getSamplingGrouping()), - escapeCSV(survey.getSamplingTask()))); - } catch (IOException e) { - LOG.error("Error writing CSV row: {}", e.getMessage()); - } - }); + survey -> { + try { + writer.write( + String.format( + "%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n", + escapeCSV(survey.getDateTime()), + escapeCSV(survey.getTimeStamp()), + escapeCSV(survey.getSurveyReferrer()), + escapeCSV(survey.getLanguage()), + escapeCSV(survey.getDevice()), + escapeCSV(survey.getScreener()), + escapeCSV(survey.getDept()), + escapeCSV(survey.getTheme()), + escapeCSV(survey.getThemeOther()), + escapeCSV(survey.getGrouping()), + escapeCSV(survey.getTask()), + escapeCSV(survey.getTaskOther()), + escapeCSV(survey.getTaskSatisfaction()), + escapeCSV(survey.getTaskEase()), + escapeCSV(survey.getTaskCompletion()), + escapeCSV(survey.getTaskImprove()), + escapeCSV(survey.getTaskImproveComment()), + escapeCSV(survey.getTaskWhyNot()), + escapeCSV(survey.getTaskWhyNotComment()), + escapeCSV(survey.getTaskSampling()), + escapeCSV(survey.getSamplingInvitation()), + escapeCSV(survey.getSamplingGC()), + escapeCSV(survey.getSamplingCanada()), + escapeCSV(survey.getSamplingTheme()), + escapeCSV(survey.getSamplingInstitution()), + escapeCSV(survey.getSamplingGrouping()), + escapeCSV(survey.getSamplingTask()))); + } catch (IOException e) { + LOG.error("Error writing CSV row: {}", e.getMessage()); + } + }); } writer.flush(); @@ -901,28 +915,37 @@ private Criteria buildExportCriteria(HttpServletRequest request) { taskCriteria, new Criteria().orOperator(nonEmptyCriteria.toArray(new Criteria[0])))); } - criteria = new Criteria().andOperator(criteria, - new Criteria().orOperator(commentCriteriaWithTasks.toArray(new Criteria[0]))); + criteria = + new Criteria() + .andOperator( + criteria, + new Criteria().orOperator(commentCriteriaWithTasks.toArray(new Criteria[0]))); } else { - criteria = new Criteria().andOperator(criteria, - new Criteria().orOperator(nonEmptyCriteria.toArray(new Criteria[0]))); + criteria = + new Criteria() + .andOperator( + criteria, new Criteria().orOperator(nonEmptyCriteria.toArray(new Criteria[0]))); } } else if (!combinedOrCriteria.isEmpty()) { - criteria = new Criteria().andOperator(criteria, - new Criteria().orOperator(combinedOrCriteria.toArray(new Criteria[0]))); + criteria = + new Criteria() + .andOperator( + criteria, new Criteria().orOperator(combinedOrCriteria.toArray(new Criteria[0]))); } - if (comments != null && !comments.isEmpty()) { - String escapedComment = escapeSpecialRegexCharacters(comments); - List commentCriteria = new ArrayList<>(); - commentCriteria.add(Criteria.where("taskImproveComment").regex(escapedComment, "i")); - commentCriteria.add(Criteria.where("taskWhyNotComment").regex(escapedComment, "i")); - commentCriteria.add(Criteria.where("themeOther").regex(escapedComment, "i")); - commentCriteria.add(Criteria.where("taskOther").regex(escapedComment, "i")); - - criteria = new Criteria().andOperator(criteria, - new Criteria().orOperator(commentCriteria.toArray(new Criteria[0]))); - } + if (comments != null && !comments.isEmpty()) { + String escapedComment = escapeSpecialRegexCharacters(comments); + List commentCriteria = new ArrayList<>(); + commentCriteria.add(Criteria.where("taskImproveComment").regex(escapedComment, "i")); + commentCriteria.add(Criteria.where("taskWhyNotComment").regex(escapedComment, "i")); + commentCriteria.add(Criteria.where("themeOther").regex(escapedComment, "i")); + commentCriteria.add(Criteria.where("taskOther").regex(escapedComment, "i")); + + criteria = + new Criteria() + .andOperator( + criteria, new Criteria().orOperator(commentCriteria.toArray(new Criteria[0]))); + } return criteria; } @@ -1002,7 +1025,6 @@ public List> departmentData(HttpServletRequest request) { .collect(Collectors.toList()); } - @GetMapping("/api/toptasks") public ResponseEntity getProblemsJson( @RequestParam Map requestParams, @@ -1140,24 +1162,24 @@ public ResponseEntity getProblemsJson( private Criteria applyDepartmentFilter(Criteria criteria, String department) { List variations = new ArrayList<>(); - for (Map.Entry> entry : institutionMappings.entrySet()) { - List mappingValues = entry.getValue(); - if (mappingValues.stream().anyMatch(v -> v.equalsIgnoreCase(department))) { - variations.addAll(mappingValues); - break; - } + for (Map.Entry> entry : institutionMappings.entrySet()) { + List mappingValues = entry.getValue(); + if (mappingValues.stream().anyMatch(v -> v.equalsIgnoreCase(department))) { + variations.addAll(mappingValues); + break; } - if (variations.isEmpty()) { - criteria.and("dept").regex("^" + Pattern.quote(department) + "$", "i"); - } else { - List deptCriteria = new ArrayList<>(); - for (String variation : variations) { - deptCriteria.add(Criteria.where("dept").regex("^" + Pattern.quote(variation) + "$", "i")); - } - criteria.orOperator(deptCriteria.toArray(new Criteria[0])); + } + if (variations.isEmpty()) { + criteria.and("dept").regex("^" + Pattern.quote(department) + "$", "i"); + } else { + List deptCriteria = new ArrayList<>(); + for (String variation : variations) { + deptCriteria.add(Criteria.where("dept").regex("^" + Pattern.quote(variation) + "$", "i")); } + criteria.orOperator(deptCriteria.toArray(new Criteria[0])); + } - return criteria; + return criteria; } @GetMapping(value = "/topTaskSurvey") @@ -1231,11 +1253,10 @@ public List getTaskNames( } private String escapeSpecialRegexCharacters(String input) { - if (input == null) { - return null; - } - // Escape all regex metacharacters - return input.replaceAll("([\\\\.|^$|()\\[\\]{}*+?])", "\\\\$1"); + if (input == null) { + return null; + } + // Escape all regex metacharacters + return input.replaceAll("([\\\\.|^$|()\\[\\]{}*+?])", "\\\\$1"); } - } diff --git a/src/main/java/ca/gc/tbs/controller/UserController.java b/src/main/java/ca/gc/tbs/controller/UserController.java index da926d7c..5b505c41 100644 --- a/src/main/java/ca/gc/tbs/controller/UserController.java +++ b/src/main/java/ca/gc/tbs/controller/UserController.java @@ -2,21 +2,21 @@ import ca.gc.tbs.domain.User; import ca.gc.tbs.service.UserService; -import java.util.List; import jakarta.servlet.http.HttpServletRequest; +import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.util.HtmlUtils; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.servlet.ModelAndView; import org.springframework.web.servlet.View; import org.springframework.web.servlet.mvc.support.RedirectAttributes; import org.springframework.web.servlet.view.RedirectView; -import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.util.HtmlUtils; @Controller @PreAuthorize("hasAuthority('ADMIN')") @@ -74,25 +74,25 @@ public String getData(String lang) { String institution = user.getInstitution(); String dateCreated = user.getDateCreated(); boolean enabled = user.isEnabled(); - List roles = user.getRoles() != null - ? user.getRoles().stream() - .map(ca.gc.tbs.domain.Role::getRole) - .toList() - : List.of(); + List roles = + user.getRoles() != null + ? user.getRoles().stream().map(ca.gc.tbs.domain.Role::getRole).toList() + : List.of(); - String status = isEn - ? (enabled ? "Enabled" : "Awaiting approval") - : (enabled ? "Activé" : "En attente d'approbation"); + String status = + isEn + ? (enabled ? "Enabled" : "Awaiting approval") + : (enabled ? "Activé" : "En attente d'approbation"); - String toggleLabel = isEn - ? (enabled ? "Disable" : "Enable") - : (enabled ? "Désactiver" : "Activer"); + String toggleLabel = + isEn ? (enabled ? "Disable" : "Enable") : (enabled ? "Désactiver" : "Activer"); String toggleClass = enabled ? "disableBtn" : "enableBtn"; String toggleIdPrefix = enabled ? "disable" : "enable"; String deleteLabel = isEn ? "Delete" : "Supprimer"; - builder.append(""" + builder.append( + """ %s %s @@ -105,12 +105,19 @@ public String getData(String lang) { - """.formatted( - HtmlUtils.htmlEscape(email != null ? email : ""), - HtmlUtils.htmlEscape(institution != null ? institution : ""), - HtmlUtils.htmlEscape(roles.toString()), dateCreated, status, - toggleIdPrefix, id, toggleClass, toggleLabel, - id, deleteLabel)); + """ + .formatted( + HtmlUtils.htmlEscape(email != null ? email : ""), + HtmlUtils.htmlEscape(institution != null ? institution : ""), + HtmlUtils.htmlEscape(roles.toString()), + dateCreated, + status, + toggleIdPrefix, + id, + toggleClass, + toggleLabel, + id, + deleteLabel)); } return builder.toString(); } catch (Exception e) { diff --git a/src/main/java/ca/gc/tbs/domain/BadWordEntry.java b/src/main/java/ca/gc/tbs/domain/BadWordEntry.java index c782b6af..6ee9fb95 100644 --- a/src/main/java/ca/gc/tbs/domain/BadWordEntry.java +++ b/src/main/java/ca/gc/tbs/domain/BadWordEntry.java @@ -6,88 +6,92 @@ import org.springframework.data.mongodb.core.mapping.Document; /** - * Entity representing a word entry in the badwords collection. - * Supports profanity filtering, threat detection, allowed words, and error keywords. + * Entity representing a word entry in the badwords collection. Supports profanity filtering, threat + * detection, allowed words, and error keywords. */ @Document(collection = "badwords") @CompoundIndex(def = "{'type': 1, 'active': 1}") @CompoundIndex(def = "{'type': 1, 'active': 1, 'language': 1}") public class BadWordEntry { - - @Id - private String id; - - @Indexed - private String word; - - @Indexed - private String language; // "en", "fr", or "both" - - @Indexed - private String type; // "profanity", "threat", "allowed", "error" - - @Indexed - private Boolean active; // true/false to enable/disable words - + + @Id private String id; + + @Indexed private String word; + + @Indexed private String language; // "en", "fr", or "both" + + @Indexed private String type; // "profanity", "threat", "allowed", "error" + + @Indexed private Boolean active; // true/false to enable/disable words + public BadWordEntry() {} - + public BadWordEntry(String word, String language, String type, Boolean active) { this.word = word; this.language = language; this.type = type; this.active = active; } - + // Getters and Setters - + public String getId() { return id; } - + public void setId(String id) { this.id = id; } - + public String getWord() { return word; } - + public void setWord(String word) { this.word = word; } - + public String getLanguage() { return language; } - + public void setLanguage(String language) { this.language = language; } - + public String getType() { return type; } - + public void setType(String type) { this.type = type; } - + public Boolean getActive() { return active; } - + public void setActive(Boolean active) { this.active = active; } - + @Override public String toString() { - return "BadWordEntry{" + - "id='" + id + '\'' + - ", word='" + word + '\'' + - ", language='" + language + '\'' + - ", type='" + type + '\'' + - ", active=" + active + - '}'; + return "BadWordEntry{" + + "id='" + + id + + '\'' + + ", word='" + + word + + '\'' + + ", language='" + + language + + '\'' + + ", type='" + + type + + '\'' + + ", active=" + + active + + '}'; } } diff --git a/src/main/java/ca/gc/tbs/domain/Problem.java b/src/main/java/ca/gc/tbs/domain/Problem.java index a5099cfe..31af3954 100644 --- a/src/main/java/ca/gc/tbs/domain/Problem.java +++ b/src/main/java/ca/gc/tbs/domain/Problem.java @@ -11,8 +11,7 @@ public class Problem { @Indexed private String url; private int urlEntries; private String problemDetails; - @Indexed - private String language; + @Indexed private String language; @Indexed private String problemDate; @Indexed private String timeStamp; @Indexed private String title; diff --git a/src/main/java/ca/gc/tbs/domain/TopTaskSurvey.java b/src/main/java/ca/gc/tbs/domain/TopTaskSurvey.java index e0096df0..3cccfba9 100644 --- a/src/main/java/ca/gc/tbs/domain/TopTaskSurvey.java +++ b/src/main/java/ca/gc/tbs/domain/TopTaskSurvey.java @@ -2,7 +2,6 @@ import org.springframework.data.annotation.Id; import org.springframework.data.mongodb.core.index.CompoundIndex; -import org.springframework.data.mongodb.core.index.Indexed; import org.springframework.data.mongodb.core.mapping.Document; @Document(collection = "toptasksurvey") diff --git a/src/main/java/ca/gc/tbs/filter/GcIpFilter.java b/src/main/java/ca/gc/tbs/filter/GcIpFilter.java index b3158cfe..edd09149 100644 --- a/src/main/java/ca/gc/tbs/filter/GcIpFilter.java +++ b/src/main/java/ca/gc/tbs/filter/GcIpFilter.java @@ -1,13 +1,6 @@ package ca.gc.tbs.filter; import ca.gc.tbs.service.GcIpValidationService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.core.annotation.Order; -import org.springframework.stereotype.Component; - import jakarta.servlet.*; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; @@ -17,221 +10,220 @@ import java.io.IOException; import java.util.HashSet; import java.util.Set; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; /** - * Filter to restrict access to Government of Canada IP addresses only - * This filter runs before authentication to ensure only GC networks can access the application + * Filter to restrict access to Government of Canada IP addresses only This filter runs before + * authentication to ensure only GC networks can access the application */ @Component @Order(1) // Run before other filters public class GcIpFilter implements Filter { - private static final Logger logger = LoggerFactory.getLogger(GcIpFilter.class); + private static final Logger logger = LoggerFactory.getLogger(GcIpFilter.class); - @Autowired - private GcIpValidationService gcIpValidationService; + @Autowired private GcIpValidationService gcIpValidationService; - @Value("${gc.ip.filter.enabled:true}") - private boolean filterEnabled; + @Value("${gc.ip.filter.enabled:true}") + private boolean filterEnabled; - @Value("${gc.ip.filter.whitelist:}") - private String whitelistIps; + @Value("${gc.ip.filter.whitelist:}") + private String whitelistIps; - @Value("${gc.ip.filter.whitelist.file:}") - private String whitelistFilePath; + @Value("${gc.ip.filter.whitelist.file:}") + private String whitelistFilePath; - // Deployment-specific: controls how the real client IP is extracted. - // X_REAL_IP — nginx/Kubernetes ingress (default for this repo) - // X_FORWARDED_FOR_LAST — AWS ALB/ECS: add GC_IP_FILTER_CLIENT_IP_SOURCE=X_FORWARDED_FOR_LAST - // to container_environment in terragrunt/aws/ecs/ecs.tf when porting - // REMOTE_ADDR — no proxy (direct connection / local) - @Value("${gc.ip.filter.client-ip-source:X_REAL_IP}") - private String clientIpSource; + // Deployment-specific: controls how the real client IP is extracted. + // X_REAL_IP — nginx/Kubernetes ingress (default for this repo) + // X_FORWARDED_FOR_LAST — AWS ALB/ECS: add GC_IP_FILTER_CLIENT_IP_SOURCE=X_FORWARDED_FOR_LAST + // to container_environment in terragrunt/aws/ecs/ecs.tf when porting + // REMOTE_ADDR — no proxy (direct connection / local) + @Value("${gc.ip.filter.client-ip-source:X_REAL_IP}") + private String clientIpSource; - private Set fileWhitelistIps = new HashSet<>(); + private Set fileWhitelistIps = new HashSet<>(); - @Override - public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) - throws IOException, ServletException { + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) + throws IOException, ServletException { - HttpServletRequest httpRequest = (HttpServletRequest) request; - HttpServletResponse httpResponse = (HttpServletResponse) response; + HttpServletRequest httpRequest = (HttpServletRequest) request; + HttpServletResponse httpResponse = (HttpServletResponse) response; - // Skip filter if disabled (for local development) - if (!filterEnabled) { - logger.debug("GC IP filter is disabled"); - chain.doFilter(request, response); - return; - } + // Skip filter if disabled (for local development) + if (!filterEnabled) { + logger.debug("GC IP filter is disabled"); + chain.doFilter(request, response); + return; + } - // Skip filter for health check endpoints (ALB health checks) - String requestPath = httpRequest.getRequestURI(); - if ("/health".equals(requestPath) || "/actuator/health".equals(requestPath)) { - logger.debug("Skipping GC IP filter for health check endpoint: {}", requestPath); - chain.doFilter(request, response); - return; - } + // Skip filter for health check endpoints (ALB health checks) + String requestPath = httpRequest.getRequestURI(); + if ("/health".equals(requestPath) || "/actuator/health".equals(requestPath)) { + logger.debug("Skipping GC IP filter for health check endpoint: {}", requestPath); + chain.doFilter(request, response); + return; + } - String clientIp = getClientIpAddress(httpRequest); - logger.debug("Request from IP: {}", clientIp); + String clientIp = getClientIpAddress(httpRequest); + logger.debug("Request from IP: {}", clientIp); - // Check if IP is in whitelist - if (isWhitelisted(clientIp)) { - logger.debug("IP {} is authorized (whitelisted)", clientIp); - chain.doFilter(request, response); - return; - } + // Check if IP is in whitelist + if (isWhitelisted(clientIp)) { + logger.debug("IP {} is authorized (whitelisted)", clientIp); + chain.doFilter(request, response); + return; + } - // Check if IP is owned by GC - if (gcIpValidationService.isGcIp(clientIp)) { - logger.debug("IP {} is authorized (GC-owned)", clientIp); - chain.doFilter(request, response); - } else { - logger.warn("Access denied for non-GC IP: {}", clientIp); - httpResponse.setStatus(HttpServletResponse.SC_FORBIDDEN); - httpResponse.setContentType("text/html; charset=UTF-8"); - httpResponse.getWriter().write( - "" + - "" + - "" + - " " + - " " + - " Access Denied" + - " " + - "" + - "" + - "

Access Denied

" + - "

This application is only accessible from Government of Canada networks.

" + - "

If you believe you should have access, please contact your system administrator.

" + - "" + - "" - ); + // Check if IP is owned by GC + if (gcIpValidationService.isGcIp(clientIp)) { + logger.debug("IP {} is authorized (GC-owned)", clientIp); + chain.doFilter(request, response); + } else { + logger.warn("Access denied for non-GC IP: {}", clientIp); + httpResponse.setStatus(HttpServletResponse.SC_FORBIDDEN); + httpResponse.setContentType("text/html; charset=UTF-8"); + httpResponse + .getWriter() + .write( + " " + + " Access Denied " + + "

Access Denied

This application is only accessible" + + " from Government of Canada networks.

If you believe you should have" + + " access, please contact your system administrator.

"); + } + } + + /** + * Extract the real client IP using the strategy configured by gc.ip.filter.client-ip-source. + * + *

X-Forwarded-For is intentionally NOT used as a generic source: it is a client-controlled + * header and trusting it directly allows an attacker to spoof any IP. + */ + private String getClientIpAddress(HttpServletRequest request) { + return switch (clientIpSource) { + case "X_REAL_IP" -> { + // X-Real-IP is set by nginx ingress to the verified TCP source IP. + // It is not forwarded from the client and cannot be spoofed. + String xRealIp = request.getHeader("X-Real-IP"); + yield (xRealIp != null + && !xRealIp.trim().isEmpty() + && !"unknown".equalsIgnoreCase(xRealIp.trim())) + ? xRealIp.trim() + : request.getRemoteAddr(); + } + case "X_FORWARDED_FOR_LAST" -> { + // AWS ALB always appends the real client IP as the rightmost entry. + // Taking the last value is safe because only the ALB can append to the right. + String xff = request.getHeader("X-Forwarded-For"); + if (xff != null && !xff.isEmpty()) { + String[] parts = xff.split(","); + String last = parts[parts.length - 1].trim(); + if (!last.isEmpty() && !"unknown".equalsIgnoreCase(last)) { + yield last; + } } + yield request.getRemoteAddr(); + } + // REMOTE_ADDR or any unrecognised value — direct connection, no proxy. + default -> request.getRemoteAddr(); + }; + } + + /** Check if IP is in the whitelist (property or file-based) */ + private boolean isWhitelisted(String ip) { + // Check property-based whitelist + if (whitelistIps != null && !whitelistIps.trim().isEmpty()) { + String[] whitelist = whitelistIps.split(","); + for (String whitelistedIp : whitelist) { + if (whitelistedIp.trim().equals(ip)) { + return true; + } + } } - /** - * Extract the real client IP using the strategy configured by gc.ip.filter.client-ip-source. - * - * X-Forwarded-For is intentionally NOT used as a generic source: it is a client-controlled - * header and trusting it directly allows an attacker to spoof any IP. - */ - private String getClientIpAddress(HttpServletRequest request) { - return switch (clientIpSource) { - case "X_REAL_IP" -> { - // X-Real-IP is set by nginx ingress to the verified TCP source IP. - // It is not forwarded from the client and cannot be spoofed. - String xRealIp = request.getHeader("X-Real-IP"); - yield (xRealIp != null && !xRealIp.trim().isEmpty() && !"unknown".equalsIgnoreCase(xRealIp.trim())) - ? xRealIp.trim() - : request.getRemoteAddr(); - } - case "X_FORWARDED_FOR_LAST" -> { - // AWS ALB always appends the real client IP as the rightmost entry. - // Taking the last value is safe because only the ALB can append to the right. - String xff = request.getHeader("X-Forwarded-For"); - if (xff != null && !xff.isEmpty()) { - String[] parts = xff.split(","); - String last = parts[parts.length - 1].trim(); - if (!last.isEmpty() && !"unknown".equalsIgnoreCase(last)) { - yield last; - } - } - yield request.getRemoteAddr(); - } - // REMOTE_ADDR or any unrecognised value — direct connection, no proxy. - default -> request.getRemoteAddr(); - }; + // Check file-based whitelist + if (!fileWhitelistIps.isEmpty() && fileWhitelistIps.contains(ip)) { + return true; } - /** - * Check if IP is in the whitelist (property or file-based) - */ - private boolean isWhitelisted(String ip) { - // Check property-based whitelist - if (whitelistIps != null && !whitelistIps.trim().isEmpty()) { - String[] whitelist = whitelistIps.split(","); - for (String whitelistedIp : whitelist) { - if (whitelistedIp.trim().equals(ip)) { - return true; - } - } - } + return false; + } - // Check file-based whitelist - if (!fileWhitelistIps.isEmpty() && fileWhitelistIps.contains(ip)) { - return true; - } - - return false; + /** + * Load whitelist IPs from file Supports comments (lines starting with #) Supports both + * comma-separated and newline-separated formats + */ + private void loadWhitelistFromFile() { + if (whitelistFilePath == null || whitelistFilePath.trim().isEmpty()) { + return; } - /** - * Load whitelist IPs from file - * Supports comments (lines starting with #) - * Supports both comma-separated and newline-separated formats - */ - private void loadWhitelistFromFile() { - if (whitelistFilePath == null || whitelistFilePath.trim().isEmpty()) { - return; - } + File file = new File(whitelistFilePath); + if (!file.exists()) { + logger.warn("Whitelist file not found: {}", whitelistFilePath); + return; + } - File file = new File(whitelistFilePath); - if (!file.exists()) { - logger.warn("Whitelist file not found: {}", whitelistFilePath); - return; + fileWhitelistIps.clear(); + try (BufferedReader reader = new BufferedReader(new FileReader(file))) { + String line; + int lineNumber = 0; + while ((line = reader.readLine()) != null) { + lineNumber++; + line = line.trim(); + + // Skip empty lines and comments + if (line.isEmpty() || line.startsWith("#")) { + continue; } - fileWhitelistIps.clear(); - try (BufferedReader reader = new BufferedReader(new FileReader(file))) { - String line; - int lineNumber = 0; - while ((line = reader.readLine()) != null) { - lineNumber++; - line = line.trim(); - - // Skip empty lines and comments - if (line.isEmpty() || line.startsWith("#")) { - continue; - } - - // Handle comma-separated IPs - if (line.contains(",")) { - String[] ips = line.split(","); - for (String ip : ips) { - String trimmedIp = ip.trim(); - if (!trimmedIp.isEmpty()) { - fileWhitelistIps.add(trimmedIp); - } - } - } else { - // Single IP per line - fileWhitelistIps.add(line); - } + // Handle comma-separated IPs + if (line.contains(",")) { + String[] ips = line.split(","); + for (String ip : ips) { + String trimmedIp = ip.trim(); + if (!trimmedIp.isEmpty()) { + fileWhitelistIps.add(trimmedIp); } - logger.info("Loaded {} IP addresses from whitelist file: {}", - fileWhitelistIps.size(), whitelistFilePath); - } catch (IOException e) { - logger.error("Error reading whitelist file {}: {}", whitelistFilePath, e.getMessage()); + } + } else { + // Single IP per line + fileWhitelistIps.add(line); } + } + logger.info( + "Loaded {} IP addresses from whitelist file: {}", + fileWhitelistIps.size(), + whitelistFilePath); + } catch (IOException e) { + logger.error("Error reading whitelist file {}: {}", whitelistFilePath, e.getMessage()); } - - @Override - public void init(FilterConfig filterConfig) throws ServletException { - // Load IPs from file if configured - loadWhitelistFromFile(); - - logger.info("GC IP Filter initialized - filter enabled: {}, client-ip-source: {}, property whitelist: {}, file whitelist: {} IPs", - filterEnabled, - clientIpSource, - whitelistIps != null && !whitelistIps.isEmpty() ? "configured" : "none", - fileWhitelistIps.size()); - } - - @Override - public void destroy() { - logger.info("GC IP Filter destroyed"); - } + } + + @Override + public void init(FilterConfig filterConfig) throws ServletException { + // Load IPs from file if configured + loadWhitelistFromFile(); + + logger.info( + "GC IP Filter initialized - filter enabled: {}, client-ip-source: {}, property whitelist:" + + " {}, file whitelist: {} IPs", + filterEnabled, + clientIpSource, + whitelistIps != null && !whitelistIps.isEmpty() ? "configured" : "none", + fileWhitelistIps.size()); + } + + @Override + public void destroy() { + logger.info("GC IP Filter destroyed"); + } } diff --git a/src/main/java/ca/gc/tbs/filter/LanguageFilter.java b/src/main/java/ca/gc/tbs/filter/LanguageFilter.java index ef197a30..6bb60bb3 100644 --- a/src/main/java/ca/gc/tbs/filter/LanguageFilter.java +++ b/src/main/java/ca/gc/tbs/filter/LanguageFilter.java @@ -1,10 +1,10 @@ package ca.gc.tbs.filter; -import java.io.IOException; -import java.util.regex.Pattern; import jakarta.servlet.*; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpSession; +import java.io.IOException; +import java.util.regex.Pattern; import org.springframework.context.i18n.LocaleContextHolder; import org.springframework.stereotype.Component; diff --git a/src/main/java/ca/gc/tbs/repository/BadWordEntryRepository.java b/src/main/java/ca/gc/tbs/repository/BadWordEntryRepository.java index 8d745494..6c58969a 100644 --- a/src/main/java/ca/gc/tbs/repository/BadWordEntryRepository.java +++ b/src/main/java/ca/gc/tbs/repository/BadWordEntryRepository.java @@ -1,58 +1,56 @@ package ca.gc.tbs.repository; +import ca.gc.tbs.domain.BadWordEntry; import java.util.List; - import org.springframework.data.mongodb.repository.MongoRepository; import org.springframework.stereotype.Repository; -import ca.gc.tbs.domain.BadWordEntry; - /** - * Repository for managing BadWordEntry entities in MongoDB. - * Provides methods to query words by type and active status. + * Repository for managing BadWordEntry entities in MongoDB. Provides methods to query words by type + * and active status. */ @Repository public interface BadWordEntryRepository extends MongoRepository { - + /** * Find all active words of a specific type. - * + * * @param type The type of words to find (profanity, threat, allowed, error) * @param active Whether the words should be active * @return List of matching BadWordEntry entities */ List findByTypeAndActive(String type, Boolean active); - + /** * Find all words of a specific type regardless of active status. - * + * * @param type The type of words to find * @return List of matching BadWordEntry entities */ List findByType(String type); - + /** * Find all active words. - * + * * @param active Whether the words should be active * @return List of matching BadWordEntry entities */ List findByActive(Boolean active); - + /** * Find all active words of a specific type and language. - * + * * @param type The type of words to find * @param language The language ("en", "fr", or "both") * @param active Whether the words should be active * @return List of matching BadWordEntry entities */ List findByTypeAndLanguageAndActive(String type, String language, Boolean active); - + /** - * Find a word entry by word text, language, and type. - * Used for duplicate detection before creating or updating entries. - * + * Find a word entry by word text, language, and type. Used for duplicate detection before + * creating or updating entries. + * * @param word The word text (case-sensitive) * @param language The language ("en", "fr", or "both") * @param type The type of word (profanity, threat, allowed, error) diff --git a/src/main/java/ca/gc/tbs/repository/CustomTopTaskRepository.java b/src/main/java/ca/gc/tbs/repository/CustomTopTaskRepository.java index 35a5cf23..ed077177 100644 --- a/src/main/java/ca/gc/tbs/repository/CustomTopTaskRepository.java +++ b/src/main/java/ca/gc/tbs/repository/CustomTopTaskRepository.java @@ -2,10 +2,10 @@ import java.util.List; import java.util.Map; - import org.springframework.data.mongodb.core.query.Criteria; public interface CustomTopTaskRepository { List findDistinctTaskCountsWithFilters(Criteria criteria); + List findTaskNamesBySearchWithFilters(String search, Criteria criteria); } diff --git a/src/main/java/ca/gc/tbs/repository/CustomTopTaskRepositoryImpl.java b/src/main/java/ca/gc/tbs/repository/CustomTopTaskRepositoryImpl.java index 6b5ad7e2..d051ad07 100644 --- a/src/main/java/ca/gc/tbs/repository/CustomTopTaskRepositoryImpl.java +++ b/src/main/java/ca/gc/tbs/repository/CustomTopTaskRepositoryImpl.java @@ -1,16 +1,14 @@ package ca.gc.tbs.repository; +import ca.gc.tbs.domain.TopTaskSurvey; // Import your domain class import java.util.List; import java.util.Map; - import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.aggregation.Aggregation; import org.springframework.data.mongodb.core.aggregation.AggregationResults; import org.springframework.data.mongodb.core.query.Criteria; -import ca.gc.tbs.domain.TopTaskSurvey; // Import your domain class - public class CustomTopTaskRepositoryImpl implements CustomTopTaskRepository { private final MongoTemplate mongoTemplate; @@ -46,15 +44,14 @@ public List findTaskNamesBySearchWithFilters(String search, Criteria cri Aggregation.newAggregation( Aggregation.match(combinedCriteria), // Apply both filter criteria and search Aggregation.group("task"), // Group by task to get distinct values - Aggregation.sort(org.springframework.data.domain.Sort.Direction.ASC, "_id") // Sort alphabetically - ); + Aggregation.sort( + org.springframework.data.domain.Sort.Direction.ASC, "_id") // Sort alphabetically + ); AggregationResults results = mongoTemplate.aggregate(aggregation, TopTaskSurvey.class, Map.class); // Extract the task names from the aggregation results - return results.getMappedResults().stream() - .map(map -> (String) map.get("_id")) - .toList(); + return results.getMappedResults().stream().map(map -> (String) map.get("_id")).toList(); } } diff --git a/src/main/java/ca/gc/tbs/repository/ProblemRepository.java b/src/main/java/ca/gc/tbs/repository/ProblemRepository.java index d3235b8c..e507ae23 100644 --- a/src/main/java/ca/gc/tbs/repository/ProblemRepository.java +++ b/src/main/java/ca/gc/tbs/repository/ProblemRepository.java @@ -1,11 +1,11 @@ package ca.gc.tbs.repository; import ca.gc.tbs.domain.Problem; +import jakarta.validation.Valid; import java.util.List; import java.util.Map; import java.util.Set; import java.util.regex.Pattern; -import jakarta.validation.Valid; import org.springframework.data.mongodb.core.aggregation.AggregationResults; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.datatables.DataTablesInput; @@ -124,6 +124,7 @@ default DataTablesOutput findAllWithErrorKeywords( "{ '$sort': { '_id': 1 } }" }) List findPageTitlesBySearch(String search); + @Aggregation( pipeline = { "{ '$match': { 'processed': 'true' } }", diff --git a/src/main/java/ca/gc/tbs/security/JWTFilter.java b/src/main/java/ca/gc/tbs/security/JWTFilter.java index b6c0eab0..1e5f72f9 100644 --- a/src/main/java/ca/gc/tbs/security/JWTFilter.java +++ b/src/main/java/ca/gc/tbs/security/JWTFilter.java @@ -1,12 +1,14 @@ package ca.gc.tbs.security; import ca.gc.tbs.service.UserService; -import java.io.IOException; -import java.util.List; import jakarta.servlet.FilterChain; import jakarta.servlet.ServletException; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; import org.springframework.security.core.context.SecurityContextHolder; @@ -14,8 +16,6 @@ import org.springframework.security.web.authentication.WebAuthenticationDetailsSource; import org.springframework.stereotype.Component; import org.springframework.web.filter.OncePerRequestFilter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; @Component public class JWTFilter extends OncePerRequestFilter { diff --git a/src/main/java/ca/gc/tbs/security/JWTUtil.java b/src/main/java/ca/gc/tbs/security/JWTUtil.java index 8df77ceb..11668795 100644 --- a/src/main/java/ca/gc/tbs/security/JWTUtil.java +++ b/src/main/java/ca/gc/tbs/security/JWTUtil.java @@ -62,9 +62,7 @@ private Boolean isTokenExpired(String token) { public String generateToken(UserDetails userDetails) { Map claims = new HashMap<>(); List authorities = - userDetails.getAuthorities().stream() - .map(GrantedAuthority::getAuthority) - .toList(); + userDetails.getAuthorities().stream().map(GrantedAuthority::getAuthority).toList(); claims.put("authorities", authorities); return createToken(claims, userDetails.getUsername()); } diff --git a/src/main/java/ca/gc/tbs/service/BadWords.java b/src/main/java/ca/gc/tbs/service/BadWords.java index e25252dd..0c03aadb 100644 --- a/src/main/java/ca/gc/tbs/service/BadWords.java +++ b/src/main/java/ca/gc/tbs/service/BadWords.java @@ -1,5 +1,8 @@ package ca.gc.tbs.service; +import ca.gc.tbs.domain.BadWordEntry; +import ca.gc.tbs.repository.BadWordEntryRepository; +import jakarta.annotation.PostConstruct; import java.util.Collections; import java.util.List; import java.util.Set; @@ -7,21 +10,15 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; - -import jakarta.annotation.PostConstruct; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import ca.gc.tbs.domain.BadWordEntry; -import ca.gc.tbs.repository.BadWordEntryRepository; - /** - * Service for managing bad words, profanity filtering, threat detection, and allowed words. - * Loads word lists from MongoDB into in-memory caches for fast lookup. - * Thread-safe using ConcurrentHashMap with lazy initialization fallback. + * Service for managing bad words, profanity filtering, threat detection, and allowed words. Loads + * word lists from MongoDB into in-memory caches for fast lookup. Thread-safe using + * ConcurrentHashMap with lazy initialization fallback. */ @Service public class BadWords { @@ -52,9 +49,8 @@ public BadWords(BadWordEntryRepository badWordEntryRepository) { } /** - * Ensures keywords are loaded from database. - * Uses lazy initialization - loads on first access if @PostConstruct didn't run. - * Thread-safe using double-checked locking. + * Ensures keywords are loaded from database. Uses lazy initialization - loads on first access + * if @PostConstruct didn't run. Thread-safe using double-checked locking. */ private void ensureLoaded() { if (!isLoaded) { @@ -71,8 +67,8 @@ private void ensureLoaded() { } /** - * Loads word configurations from MongoDB on service initialization. - * Called automatically by Spring after dependency injection. + * Loads word configurations from MongoDB on service initialization. Called automatically by + * Spring after dependency injection. */ @PostConstruct public void loadConfigs() { @@ -85,40 +81,50 @@ public void loadConfigs() { } // Load profanity words - List profanityEntries = badWordEntryRepository.findByTypeAndActive("profanity", true); - profanityEntries.forEach(entry -> { - String word = entry.getWord().trim().toLowerCase(); - profanityWords.add(word); - allFilterWords.add(word); - }); + List profanityEntries = + badWordEntryRepository.findByTypeAndActive("profanity", true); + profanityEntries.forEach( + entry -> { + String word = entry.getWord().trim().toLowerCase(); + profanityWords.add(word); + allFilterWords.add(word); + }); // Load threat words List threatEntries = badWordEntryRepository.findByTypeAndActive("threat", true); - threatEntries.forEach(entry -> { - String word = entry.getWord().trim().toLowerCase(); - threatWords.add(word); - allFilterWords.add(word); - }); + threatEntries.forEach( + entry -> { + String word = entry.getWord().trim().toLowerCase(); + threatWords.add(word); + allFilterWords.add(word); + }); // Load allowed words - List allowedEntries = badWordEntryRepository.findByTypeAndActive("allowed", true); - allowedEntries.forEach(entry -> { - String word = entry.getWord().trim().toLowerCase(); - allowedWords.add(word); - }); + List allowedEntries = + badWordEntryRepository.findByTypeAndActive("allowed", true); + allowedEntries.forEach( + entry -> { + String word = entry.getWord().trim().toLowerCase(); + allowedWords.add(word); + }); // Load error keywords List errorEntries = badWordEntryRepository.findByTypeAndActive("error", true); - errorEntries.forEach(entry -> { - String word = entry.getWord().trim().toLowerCase(); - errorKeywords.add(word); - }); + errorEntries.forEach( + entry -> { + String word = entry.getWord().trim().toLowerCase(); + errorKeywords.add(word); + }); // Compile the filter pattern after all words are loaded compileFilterPattern(); - logger.info("Loaded {} profanity, {} threat, {} allowed, {} error keywords", - profanityWords.size(), threatWords.size(), allowedWords.size(), errorKeywords.size()); + logger.info( + "Loaded {} profanity, {} threat, {} allowed, {} error keywords", + profanityWords.size(), + threatWords.size(), + allowedWords.size(), + errorKeywords.size()); isLoaded = true; @@ -173,7 +179,8 @@ private void compileFilterPattern() { filterPattern = null; return; } - String patternString = allFilterWords.stream() + String patternString = + allFilterWords.stream() .filter(word -> word != null && !word.trim().isEmpty()) .map(Pattern::quote) .map(word -> "\\b" + word + "\\b") @@ -182,8 +189,8 @@ private void compileFilterPattern() { } /** - * Censors profanity and threats in the given text by replacing them with asterisks. - * Words in the allowed words list are never censored. + * Censors profanity and threats in the given text by replacing them with asterisks. Words in the + * allowed words list are never censored. * * @param text The text to censor * @return The censored text @@ -214,16 +221,14 @@ public String censor(String text) { return result.toString(); } - /** - * Creates a mask of asterisks for a given word. - */ + /** Creates a mask of asterisks for a given word. */ private String createMask(String word) { return word.replaceAll(".", "*"); } /** - * Reloads word configurations from MongoDB. - * Useful for refreshing the cache without restarting the application. + * Reloads word configurations from MongoDB. Useful for refreshing the cache without restarting + * the application. */ public void reload() { logger.info("Reloading word configurations from MongoDB..."); diff --git a/src/main/java/ca/gc/tbs/service/ContentService.java b/src/main/java/ca/gc/tbs/service/ContentService.java index 6d30027f..b70a1254 100644 --- a/src/main/java/ca/gc/tbs/service/ContentService.java +++ b/src/main/java/ca/gc/tbs/service/ContentService.java @@ -1,28 +1,24 @@ package ca.gc.tbs.service; +import jakarta.annotation.PostConstruct; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; -import java.util.List; import java.util.Set; import java.util.regex.Pattern; - -import jakarta.annotation.PostConstruct; - -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - import opennlp.tools.namefind.NameFinderME; import opennlp.tools.namefind.TokenNameFinderModel; import opennlp.tools.tokenize.TokenizerME; import opennlp.tools.tokenize.TokenizerModel; import opennlp.tools.util.Span; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; @Service public class ContentService { @@ -40,33 +36,38 @@ public class ContentService { Pattern.compile( "(?:(?:\\+?1\\s*(?:[.-]\\s*)?)?(?:\\(\\s*([2-9]1[02-9]|[2-9][02-8]1|[2-9][02-8][02-9])\\s*\\)|([2-9]1[02-9]|[2-9][02-8]1|[2-9][02-8][02-9]))\\s*(?:[.-]\\s*)?)?([2-9]1[02-9]|[2-9][02-9]1|[2-9][02-9]{2})\\s*(?:[.-]\\s*)?([0-9]{4})(?:\\s*(?:#|x\\.?|ext\\.?|extension)\\s*(\\d+))?"); private static final Pattern EMAIL_PATTERN = - Pattern.compile("([a-zA-Z0-9_+\\-\\.]+)\\s*@\\s*([a-zA-Z0-9_\\-\\.]+)(?:\\s*[\\.,]\\s*([a-zA-Z]{0,10}))?"); + Pattern.compile( + "([a-zA-Z0-9_+\\-\\.]+)\\s*@\\s*([a-zA-Z0-9_\\-\\.]+)(?:\\s*[\\.,]\\s*([a-zA-Z]{0,10}))?"); // Address patterns for English and French street addresses (Patterns A, B, C, D) // Pattern A: NUMBER + (DIRECTION) + WORD(S) + SUFFIX + (DIRECTION) private static final Pattern ADDRESS_PATTERN_1 = - Pattern.compile("(?i)\\b(\\d{1,6}[A-Za-zÀ-ÿ]?)\\s+" + - "(?:(?:n|s|e|w|ne|nw|se|sw|o|no|so|north|south|east|west)\\s+)?" + - "(?:(?:\\d{1,2}(?:st|nd|rd|th)|[A-Za-zÀ-ÿ][A-Za-zÀ-ÿ''\\-]*)(?:\\s+(?:\\d{1,2}(?:st|nd|rd|th)|[A-Za-zÀ-ÿ][A-Za-zÀ-ÿ''\\-]*)){0,3})\\s+" + - "(?:street|avenue|road|drive|boulevard|lane|court|place|terrace|parkway|circle|highway|way|loop|trail|pike|row|crescent|close|point|green|grove|gate|heights|landing|link|manor|park|ridge|rise|square|view|walk|crossing|meadow|garden|gardens|glen|heath|hollow|knoll|mews|village|shore|shores|hill|hills|acres|valley|rue|chemin|route|terrasse|rang|promenade|cours|voie|terrain|all[ée]e?|st|ave|av|av\\.|rd|dr|blvd|boul|boul\\.|ln|ct|pl|ter|terr|pkwy|cir|ci|hwy|wy|trl|cres|cr|cl|pt|gr|gv|ga|ht|hts|ld|lk|mr|pa|pk|rg|ri|rw|sq|tc|vi|vw|wk|co|ba|bv|hl|tr|cv|li|me|gd|mt|ca|gw|ce|he|sm|rp|al|ch|ch\\.|chem|chem\\.|rte|all\\.|allee|prom|prom\\.)" + - "(?:\\s+(?:n|s|e|w|ne|nw|se|sw|o|no|so|north|south|east|west))?\\b"); + Pattern.compile( + "(?i)\\b(\\d{1,6}[A-Za-zÀ-ÿ]?)\\s+" + + "(?:(?:n|s|e|w|ne|nw|se|sw|o|no|so|north|south|east|west)\\s+)?" + + "(?:(?:\\d{1,2}(?:st|nd|rd|th)|[A-Za-zÀ-ÿ][A-Za-zÀ-ÿ''\\-]*)(?:\\s+(?:\\d{1,2}(?:st|nd|rd|th)|[A-Za-zÀ-ÿ][A-Za-zÀ-ÿ''\\-]*)){0,3})\\s+" + + "(?:street|avenue|road|drive|boulevard|lane|court|place|terrace|parkway|circle|highway|way|loop|trail|pike|row|crescent|close|point|green|grove|gate|heights|landing|link|manor|park|ridge|rise|square|view|walk|crossing|meadow|garden|gardens|glen|heath|hollow|knoll|mews|village|shore|shores|hill|hills|acres|valley|rue|chemin|route|terrasse|rang|promenade|cours|voie|terrain|all[ée]e?|st|ave|av|av\\.|rd|dr|blvd|boul|boul\\.|ln|ct|pl|ter|terr|pkwy|cir|ci|hwy|wy|trl|cres|cr|cl|pt|gr|gv|ga|ht|hts|ld|lk|mr|pa|pk|rg|ri|rw|sq|tc|vi|vw|wk|co|ba|bv|hl|tr|cv|li|me|gd|mt|ca|gw|ce|he|sm|rp|al|ch|ch\\.|chem|chem\\.|rte|all\\.|allee|prom|prom\\.)" + + "(?:\\s+(?:n|s|e|w|ne|nw|se|sw|o|no|so|north|south|east|west))?\\b"); // Pattern B: NUMBER + SUFFIX + (FR_ARTICLES) + WORD(S) private static final Pattern ADDRESS_PATTERN_2 = - Pattern.compile("(?i)\\b(\\d{1,6}[A-Za-zÀ-ÿ]?),?\\s+" + - "(?:street|avenue|road|drive|boulevard|lane|court|place|terrace|parkway|circle|highway|way|loop|trail|pike|row|crescent|close|point|green|grove|gate|heights|landing|link|manor|park|ridge|rise|square|view|walk|crossing|meadow|garden|gardens|glen|heath|hollow|knoll|mews|village|shore|shores|hill|hills|acres|valley|rue|chemin|route|terrasse|rang|promenade|cours|voie|terrain|all[ée]e?|st|ave|av|av\\.|rd|dr|blvd|boul|boul\\.|ln|ct|pl|ter|terr|pkwy|cir|ci|hwy|wy|trl|cres|cr|cl|pt|gr|gv|ga|ht|hts|ld|lk|mr|pa|pk|rg|ri|rw|sq|tc|vi|vw|wk|co|ba|bv|hl|tr|cv|li|me|gd|mt|ca|gw|ce|he|sm|rp|al|ch|ch\\.|chem|chem\\.|rte|all\\.|allee|prom|prom\\.)\\s+" + - "(?:de\\s+la\\s+|du\\s+|des\\s+|de\\s+|le\\s+|la\\s+|les\\s+|d'|l')?" + - "(?:(?:\\d{1,2}(?:st|nd|rd|th)|[A-Za-zÀ-ÿ][A-Za-zÀ-ÿ''\\-]*)(?:\\s+(?:\\d{1,2}(?:st|nd|rd|th)|[A-Za-zÀ-ÿ][A-Za-zÀ-ÿ''\\-]*)){0,3})\\b"); + Pattern.compile( + "(?i)\\b(\\d{1,6}[A-Za-zÀ-ÿ]?),?\\s+" + + "(?:street|avenue|road|drive|boulevard|lane|court|place|terrace|parkway|circle|highway|way|loop|trail|pike|row|crescent|close|point|green|grove|gate|heights|landing|link|manor|park|ridge|rise|square|view|walk|crossing|meadow|garden|gardens|glen|heath|hollow|knoll|mews|village|shore|shores|hill|hills|acres|valley|rue|chemin|route|terrasse|rang|promenade|cours|voie|terrain|all[ée]e?|st|ave|av|av\\.|rd|dr|blvd|boul|boul\\.|ln|ct|pl|ter|terr|pkwy|cir|ci|hwy|wy|trl|cres|cr|cl|pt|gr|gv|ga|ht|hts|ld|lk|mr|pa|pk|rg|ri|rw|sq|tc|vi|vw|wk|co|ba|bv|hl|tr|cv|li|me|gd|mt|ca|gw|ce|he|sm|rp|al|ch|ch\\.|chem|chem\\.|rte|all\\.|allee|prom|prom\\.)\\s+" + + "(?:de\\s+la\\s+|du\\s+|des\\s+|de\\s+|le\\s+|la\\s+|les\\s+|d'|l')?" + + "(?:(?:\\d{1,2}(?:st|nd|rd|th)|[A-Za-zÀ-ÿ][A-Za-zÀ-ÿ''\\-]*)(?:\\s+(?:\\d{1,2}(?:st|nd|rd|th)|[A-Za-zÀ-ÿ][A-Za-zÀ-ÿ''\\-]*)){0,3})\\b"); // Pattern C: NUMBER + SUFFIX private static final Pattern ADDRESS_PATTERN_3 = - Pattern.compile("(?i)\\b(\\d{1,6}[A-Za-zÀ-ÿ]?)\\s+" + - "(?:street|avenue|road|drive|boulevard|lane|court|place|terrace|parkway|circle|highway|way|loop|trail|pike|row|crescent|close|point|green|grove|gate|heights|landing|link|manor|park|ridge|rise|square|view|walk|crossing|meadow|garden|gardens|glen|heath|hollow|knoll|mews|village|shore|shores|hill|hills|acres|valley|rue|chemin|route|terrasse|rang|promenade|cours|voie|terrain|all[ée]e?|st|ave|av|av\\.|rd|dr|blvd|boul|boul\\.|ln|ct|pl|ter|terr|pkwy|cir|ci|hwy|wy|trl|cres|cr|cl|pt|gr|gv|ga|ht|hts|ld|lk|mr|pa|pk|rg|ri|rw|sq|tc|vi|vw|wk|co|ba|bv|hl|tr|cv|li|me|gd|mt|ca|gw|ce|he|sm|rp|al|ch|ch\\.|chem|chem\\.|rte|all\\.|allee|prom|prom\\.)\\b"); + Pattern.compile( + "(?i)\\b(\\d{1,6}[A-Za-zÀ-ÿ]?)\\s+" + + "(?:street|avenue|road|drive|boulevard|lane|court|place|terrace|parkway|circle|highway|way|loop|trail|pike|row|crescent|close|point|green|grove|gate|heights|landing|link|manor|park|ridge|rise|square|view|walk|crossing|meadow|garden|gardens|glen|heath|hollow|knoll|mews|village|shore|shores|hill|hills|acres|valley|rue|chemin|route|terrasse|rang|promenade|cours|voie|terrain|all[ée]e?|st|ave|av|av\\.|rd|dr|blvd|boul|boul\\.|ln|ct|pl|ter|terr|pkwy|cir|ci|hwy|wy|trl|cres|cr|cl|pt|gr|gv|ga|ht|hts|ld|lk|mr|pa|pk|rg|ri|rw|sq|tc|vi|vw|wk|co|ba|bv|hl|tr|cv|li|me|gd|mt|ca|gw|ce|he|sm|rp|al|ch|ch\\.|chem|chem\\.|rte|all\\.|allee|prom|prom\\.)\\b"); // Pattern D: SUFFIX + NUMBER private static final Pattern ADDRESS_PATTERN_4 = - Pattern.compile("(?i)\\b(?:street|avenue|road|drive|boulevard|lane|court|place|terrace|parkway|circle|highway|way|loop|trail|pike|row|crescent|close|point|green|grove|gate|heights|landing|link|manor|park|ridge|rise|square|view|walk|crossing|meadow|garden|gardens|glen|heath|hollow|knoll|mews|village|shore|shores|hill|hills|acres|valley|rue|chemin|route|terrasse|rang|promenade|cours|voie|terrain|all[ée]e?|st|ave|av|av\\.|rd|dr|blvd|boul|boul\\.|ln|ct|pl|ter|terr|pkwy|cir|ci|hwy|wy|trl|cres|cr|cl|pt|gr|gv|ga|ht|hts|ld|lk|mr|pa|pk|rg|ri|rw|sq|tc|vi|vw|wk|co|ba|bv|hl|tr|cv|li|me|gd|mt|ca|gw|ce|he|sm|rp|al|ch|ch\\.|chem|chem\\.|rte|all\\.|allee|prom|prom\\.)\\s+(\\d{1,6}[A-Za-zÀ-ÿ]?)\\b"); + Pattern.compile( + "(?i)\\b(?:street|avenue|road|drive|boulevard|lane|court|place|terrace|parkway|circle|highway|way|loop|trail|pike|row|crescent|close|point|green|grove|gate|heights|landing|link|manor|park|ridge|rise|square|view|walk|crossing|meadow|garden|gardens|glen|heath|hollow|knoll|mews|village|shore|shores|hill|hills|acres|valley|rue|chemin|route|terrasse|rang|promenade|cours|voie|terrain|all[ée]e?|st|ave|av|av\\.|rd|dr|blvd|boul|boul\\.|ln|ct|pl|ter|terr|pkwy|cir|ci|hwy|wy|trl|cres|cr|cl|pt|gr|gv|ga|ht|hts|ld|lk|mr|pa|pk|rg|ri|rw|sq|tc|vi|vw|wk|co|ba|bv|hl|tr|cv|li|me|gd|mt|ca|gw|ce|he|sm|rp|al|ch|ch\\.|chem|chem\\.|rte|all\\.|allee|prom|prom\\.)\\s+(\\d{1,6}[A-Za-zÀ-ÿ]?)\\b"); // Apache OpenNLP models private TokenizerModel tokenizerModel; @@ -194,9 +195,7 @@ private String cleanStreetAddress(String content) { return content; } - /** - * Cleans person names from content using Apache OpenNLP entity recognition. - */ + /** Cleans person names from content using Apache OpenNLP entity recognition. */ public String cleanNames(String content) { if (tokenizerModel == null || nerModel == null) { logger.warn("OpenNLP models not loaded, skipping name cleaning"); diff --git a/src/main/java/ca/gc/tbs/service/DashboardService.java b/src/main/java/ca/gc/tbs/service/DashboardService.java index 91fdf052..851d463d 100644 --- a/src/main/java/ca/gc/tbs/service/DashboardService.java +++ b/src/main/java/ca/gc/tbs/service/DashboardService.java @@ -17,86 +17,91 @@ @Service public class DashboardService { - private static final Logger LOGGER = LoggerFactory.getLogger(DashboardService.class); + private static final Logger LOGGER = LoggerFactory.getLogger(DashboardService.class); - private final ProblemCacheService problemCacheService; + private final ProblemCacheService problemCacheService; - public DashboardService(ProblemCacheService problemCacheService) { - this.problemCacheService = problemCacheService; - } + public DashboardService(ProblemCacheService problemCacheService) { + this.problemCacheService = problemCacheService; + } - public record DashboardStats( - List problems, - List problemsByDate, - int totalComments, - int totalPages - ) {} + public record DashboardStats( + List problems, List problemsByDate, int totalComments, int totalPages) {} - @Cacheable(value = "dashboardStats", key = "'all'", sync = true) - public DashboardStats getDashboardStats() { - LOGGER.info("Computing DashboardStats for cache..."); - List processedProblems = problemCacheService.getProcessedProblems(); + @Cacheable(value = "dashboardStats", key = "'all'", sync = true) + public DashboardStats getDashboardStats() { + LOGGER.info("Computing DashboardStats for cache..."); + List processedProblems = problemCacheService.getProcessedProblems(); - // 1. Group raw records by (url, problemDate) - List merged = new ArrayList<>( + // 1. Group raw records by (url, problemDate) + List merged = + new ArrayList<>( processedProblems.stream() - .collect(Collectors.groupingBy( - p -> new AbstractMap.SimpleEntry<>(p.getUrl(), p.getProblemDate()), - Collectors.collectingAndThen(Collectors.toList(), list -> { - Problem p = new Problem(); - p.setUrl(list.getFirst().getUrl()); - p.setProblemDate(list.getFirst().getProblemDate()); - p.setUrlEntries(list.size()); - p.setInstitution(list.getFirst().getInstitution()); - p.setTitle(list.getFirst().getTitle()); - p.setLanguage(list.getFirst().getLanguage()); - p.setSection(list.getFirst().getSection()); - p.setTheme(list.getFirst().getTheme()); - return p; - }))) + .collect( + Collectors.groupingBy( + p -> new AbstractMap.SimpleEntry<>(p.getUrl(), p.getProblemDate()), + Collectors.collectingAndThen( + Collectors.toList(), + list -> { + Problem p = new Problem(); + p.setUrl(list.getFirst().getUrl()); + p.setProblemDate(list.getFirst().getProblemDate()); + p.setUrlEntries(list.size()); + p.setInstitution(list.getFirst().getInstitution()); + p.setTitle(list.getFirst().getTitle()); + p.setLanguage(list.getFirst().getLanguage()); + p.setSection(list.getFirst().getSection()); + p.setTheme(list.getFirst().getTheme()); + return p; + }))) .values()); - // 2. Filter out future dates - LocalDate cutoff = LocalDate.now(); - merged = merged.stream() + // 2. Filter out future dates + LocalDate cutoff = LocalDate.now(); + merged = + merged.stream() .filter(p -> isValidDate(p.getProblemDate(), DateTimeFormatter.ISO_LOCAL_DATE)) - .filter(p -> !LocalDate.parse(p.getProblemDate(), DateTimeFormatter.ISO_LOCAL_DATE).isAfter(cutoff)) + .filter( + p -> + !LocalDate.parse(p.getProblemDate(), DateTimeFormatter.ISO_LOCAL_DATE) + .isAfter(cutoff)) .collect(Collectors.toList()); - List problemsByDate = new ArrayList<>(merged); + List problemsByDate = new ArrayList<>(merged); - // 3. Merge across dates for final problem list - List fullyMerged = mergeProblemsAcrossDates(merged); - fullyMerged.sort(Comparator.comparingInt(Problem::getUrlEntries).reversed()); + // 3. Merge across dates for final problem list + List fullyMerged = mergeProblemsAcrossDates(merged); + fullyMerged.sort(Comparator.comparingInt(Problem::getUrlEntries).reversed()); - int totalComments = fullyMerged.stream().mapToInt(Problem::getUrlEntries).sum(); - int totalPages = fullyMerged.size(); + int totalComments = fullyMerged.stream().mapToInt(Problem::getUrlEntries).sum(); + int totalPages = fullyMerged.size(); - LOGGER.info("DashboardStats computed: {} comments across {} pages", totalComments, totalPages); - return new DashboardStats(fullyMerged, problemsByDate, totalComments, totalPages); - } + LOGGER.info("DashboardStats computed: {} comments across {} pages", totalComments, totalPages); + return new DashboardStats(fullyMerged, problemsByDate, totalComments, totalPages); + } - private List mergeProblemsAcrossDates(List problems) { - Map urlToProblemMap = new LinkedHashMap<>(); - for (Problem problem : problems) { - urlToProblemMap.merge( - problem.getUrl(), - problem, - (existingProblem, newProblem) -> { - Problem updatedProblem = new Problem(existingProblem); - updatedProblem.setUrlEntries(existingProblem.getUrlEntries() + newProblem.getUrlEntries()); - return updatedProblem; - }); - } - return new ArrayList<>(urlToProblemMap.values()); + private List mergeProblemsAcrossDates(List problems) { + Map urlToProblemMap = new LinkedHashMap<>(); + for (Problem problem : problems) { + urlToProblemMap.merge( + problem.getUrl(), + problem, + (existingProblem, newProblem) -> { + Problem updatedProblem = new Problem(existingProblem); + updatedProblem.setUrlEntries( + existingProblem.getUrlEntries() + newProblem.getUrlEntries()); + return updatedProblem; + }); } + return new ArrayList<>(urlToProblemMap.values()); + } - private boolean isValidDate(String value, DateTimeFormatter formatter) { - try { - LocalDate.parse(value, formatter); - return true; - } catch (Exception e) { - return false; - } + private boolean isValidDate(String value, DateTimeFormatter formatter) { + try { + LocalDate.parse(value, formatter); + return true; + } catch (Exception e) { + return false; } + } } diff --git a/src/main/java/ca/gc/tbs/service/ErrorKeywordService.java b/src/main/java/ca/gc/tbs/service/ErrorKeywordService.java index 09f92c7f..acd98d86 100644 --- a/src/main/java/ca/gc/tbs/service/ErrorKeywordService.java +++ b/src/main/java/ca/gc/tbs/service/ErrorKeywordService.java @@ -1,11 +1,11 @@ package ca.gc.tbs.service; +import jakarta.annotation.PostConstruct; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.util.*; import java.util.regex.Pattern; -import jakarta.annotation.PostConstruct; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.core.io.ClassPathResource; diff --git a/src/main/java/ca/gc/tbs/service/GcIpValidationService.java b/src/main/java/ca/gc/tbs/service/GcIpValidationService.java index fb2e6335..69bae0b8 100644 --- a/src/main/java/ca/gc/tbs/service/GcIpValidationService.java +++ b/src/main/java/ca/gc/tbs/service/GcIpValidationService.java @@ -9,113 +9,111 @@ import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; -import java.util.concurrent.TimeUnit; - /** - * Service to validate if an IP address is owned by the Government of Canada - * Uses RDAP (Registration Data Access Protocol) API to check IP ownership + * Service to validate if an IP address is owned by the Government of Canada Uses RDAP (Registration + * Data Access Protocol) API to check IP ownership */ @Service public class GcIpValidationService { - private static final Logger logger = LoggerFactory.getLogger(GcIpValidationService.class); - private static final String RDAP_API_URL = "https://rdap.arin.net/registry/ip/"; - private static final String GC_REGISTRANT_HANDLE = "SSC-299"; // Shared Services Canada handle - - private final RestTemplate restTemplate; - private final ObjectMapper objectMapper; - - public GcIpValidationService() { - SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory(); - factory.setConnectTimeout(5000); - factory.setReadTimeout(5000); - this.restTemplate = new RestTemplate(factory); - this.objectMapper = new ObjectMapper(); + private static final Logger logger = LoggerFactory.getLogger(GcIpValidationService.class); + private static final String RDAP_API_URL = "https://rdap.arin.net/registry/ip/"; + private static final String GC_REGISTRANT_HANDLE = "SSC-299"; // Shared Services Canada handle + + private final RestTemplate restTemplate; + private final ObjectMapper objectMapper; + + public GcIpValidationService() { + SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory(); + factory.setConnectTimeout(5000); + factory.setReadTimeout(5000); + this.restTemplate = new RestTemplate(factory); + this.objectMapper = new ObjectMapper(); + } + + /** + * Check if an IP address is owned by the Government of Canada Results are cached for 24 hours to + * avoid excessive RDAP API calls + * + * @param ipAddress The IP address to check + * @return true if the IP is owned by GC, false otherwise + */ + @Cacheable(value = "gcIpCache", key = "#ipAddress") + public boolean isGcIp(String ipAddress) { + if (ipAddress == null || ipAddress.isEmpty()) { + logger.warn("Received null or empty IP address"); + return false; } - /** - * Check if an IP address is owned by the Government of Canada - * Results are cached for 24 hours to avoid excessive RDAP API calls - * - * @param ipAddress The IP address to check - * @return true if the IP is owned by GC, false otherwise - */ - @Cacheable(value = "gcIpCache", key = "#ipAddress") - public boolean isGcIp(String ipAddress) { - if (ipAddress == null || ipAddress.isEmpty()) { - logger.warn("Received null or empty IP address"); - return false; - } + try { + logger.debug("Checking if IP {} is owned by GC", ipAddress); - try { - logger.debug("Checking if IP {} is owned by GC", ipAddress); - - String url = RDAP_API_URL + ipAddress; - String response = restTemplate.getForObject(url, String.class); - - if (response == null) { - logger.warn("Received null response from RDAP API for IP {}", ipAddress); - return false; - } - - JsonNode root = objectMapper.readTree(response); - JsonNode entities = root.get("entities"); - - if (entities != null && entities.isArray()) { - boolean isGc = recursiveEntitySearch(entities); - logger.info("IP {} is {} owned by GC", ipAddress, isGc ? "" : "NOT"); - return isGc; - } - - logger.warn("No entities found in RDAP response for IP {}", ipAddress); - return false; - - } catch (Exception e) { - logger.error("Error checking IP {} ownership: {}", ipAddress, e.getMessage()); - // Fail closed - if we can't verify, block access - return false; - } + String url = RDAP_API_URL + ipAddress; + String response = restTemplate.getForObject(url, String.class); + + if (response == null) { + logger.warn("Received null response from RDAP API for IP {}", ipAddress); + return false; + } + + JsonNode root = objectMapper.readTree(response); + JsonNode entities = root.get("entities"); + + if (entities != null && entities.isArray()) { + boolean isGc = recursiveEntitySearch(entities); + logger.info("IP {} is {} owned by GC", ipAddress, isGc ? "" : "NOT"); + return isGc; + } + + logger.warn("No entities found in RDAP response for IP {}", ipAddress); + return false; + + } catch (Exception e) { + logger.error("Error checking IP {} ownership: {}", ipAddress, e.getMessage()); + // Fail closed - if we can't verify, block access + return false; } + } - /** - * Recursively search through entity records to find GC registrant - * - * @param entities JsonNode array of entities - * @return true if SSC-299 (Shared Services Canada) is found in registrants - */ - private boolean recursiveEntitySearch(JsonNode entities) { - if (entities == null || !entities.isArray()) { - return false; + /** + * Recursively search through entity records to find GC registrant + * + * @param entities JsonNode array of entities + * @return true if SSC-299 (Shared Services Canada) is found in registrants + */ + private boolean recursiveEntitySearch(JsonNode entities) { + if (entities == null || !entities.isArray()) { + return false; + } + + for (JsonNode entity : entities) { + // Check if this entity is a registrant + JsonNode roles = entity.get("roles"); + if (roles != null && roles.isArray()) { + boolean isRegistrant = false; + for (JsonNode role : roles) { + if ("registrant".equals(role.asText())) { + isRegistrant = true; + break; + } } - for (JsonNode entity : entities) { - // Check if this entity is a registrant - JsonNode roles = entity.get("roles"); - if (roles != null && roles.isArray()) { - boolean isRegistrant = false; - for (JsonNode role : roles) { - if ("registrant".equals(role.asText())) { - isRegistrant = true; - break; - } - } - - // If this is a registrant, check if it's SSC-299 - if (isRegistrant) { - JsonNode handle = entity.get("handle"); - if (handle != null && GC_REGISTRANT_HANDLE.equals(handle.asText())) { - return true; - } - } - } - - // Recursively check nested entities - JsonNode nestedEntities = entity.get("entities"); - if (nestedEntities != null && recursiveEntitySearch(nestedEntities)) { - return true; - } + // If this is a registrant, check if it's SSC-299 + if (isRegistrant) { + JsonNode handle = entity.get("handle"); + if (handle != null && GC_REGISTRANT_HANDLE.equals(handle.asText())) { + return true; + } } + } - return false; + // Recursively check nested entities + JsonNode nestedEntities = entity.get("entities"); + if (nestedEntities != null && recursiveEntitySearch(nestedEntities)) { + return true; + } } + + return false; + } } diff --git a/src/main/java/ca/gc/tbs/service/ProblemCacheService.java b/src/main/java/ca/gc/tbs/service/ProblemCacheService.java index b0b77383..1e06161a 100644 --- a/src/main/java/ca/gc/tbs/service/ProblemCacheService.java +++ b/src/main/java/ca/gc/tbs/service/ProblemCacheService.java @@ -1,7 +1,8 @@ package ca.gc.tbs.service; +import ca.gc.tbs.domain.Problem; +import ca.gc.tbs.repository.ProblemRepository; import java.util.List; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.cache.annotation.CacheEvict; @@ -9,9 +10,6 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; -import ca.gc.tbs.domain.Problem; -import ca.gc.tbs.repository.ProblemRepository; - @Service public class ProblemCacheService { @@ -24,7 +22,9 @@ public ProblemCacheService(ProblemRepository problemRepository) { } @Scheduled(cron = "0 0 0 * * *") - @CacheEvict(value = {"distinctUrls", "processedProblems"}, allEntries = true) + @CacheEvict( + value = {"distinctUrls", "processedProblems"}, + allEntries = true) public void clearCacheDaily() { LOGGER.info("Evicting caches at midnight"); } diff --git a/src/main/java/ca/gc/tbs/service/ProblemDateService.java b/src/main/java/ca/gc/tbs/service/ProblemDateService.java index 10aa513a..2561b2f4 100644 --- a/src/main/java/ca/gc/tbs/service/ProblemDateService.java +++ b/src/main/java/ca/gc/tbs/service/ProblemDateService.java @@ -25,20 +25,29 @@ public Map getProblemDates() { // Determine the current fiscal quarter and calculate the date range record DateRange(LocalDate start, LocalDate end) {} - DateRange range = switch (currentMonth) { - case APRIL, MAY, JUNE -> - // Q1 (April 1 - June 30) - Show Q4 (previous year) and Q1 (current year) - new DateRange(LocalDate.of(currentYear, Month.JANUARY, 1), LocalDate.of(currentYear, Month.JUNE, 30)); - case JULY, AUGUST, SEPTEMBER -> - // Q2 (July 1 - September 30) - Show Q1 and Q2 - new DateRange(LocalDate.of(currentYear, Month.APRIL, 1), LocalDate.of(currentYear, Month.SEPTEMBER, 30)); - case OCTOBER, NOVEMBER, DECEMBER -> - // Q3 (October 1 - December 31) - Show Q2 and Q3 - new DateRange(LocalDate.of(currentYear, Month.JULY, 1), LocalDate.of(currentYear, Month.DECEMBER, 31)); - case JANUARY, FEBRUARY, MARCH -> - // Q4 (January 1 - March 31) - Show Q3 (previous year) and Q4 (current year) - new DateRange(LocalDate.of(currentYear - 1, Month.OCTOBER, 1), LocalDate.of(currentYear, Month.MARCH, 31)); - }; + DateRange range = + switch (currentMonth) { + case APRIL, MAY, JUNE -> + // Q1 (April 1 - June 30) - Show Q4 (previous year) and Q1 (current year) + new DateRange( + LocalDate.of(currentYear, Month.JANUARY, 1), + LocalDate.of(currentYear, Month.JUNE, 30)); + case JULY, AUGUST, SEPTEMBER -> + // Q2 (July 1 - September 30) - Show Q1 and Q2 + new DateRange( + LocalDate.of(currentYear, Month.APRIL, 1), + LocalDate.of(currentYear, Month.SEPTEMBER, 30)); + case OCTOBER, NOVEMBER, DECEMBER -> + // Q3 (October 1 - December 31) - Show Q2 and Q3 + new DateRange( + LocalDate.of(currentYear, Month.JULY, 1), + LocalDate.of(currentYear, Month.DECEMBER, 31)); + case JANUARY, FEBRUARY, MARCH -> + // Q4 (January 1 - March 31) - Show Q3 (previous year) and Q4 (current year) + new DateRange( + LocalDate.of(currentYear - 1, Month.OCTOBER, 1), + LocalDate.of(currentYear, Month.MARCH, 31)); + }; LocalDate earliestDate = range.start(); LocalDate latestDate = range.end(); @@ -51,7 +60,8 @@ record DateRange(LocalDate start, LocalDate end) {} return resultMap; } - // The clearCacheDaily and refreshProblemDates methods are no longer needed as dates are calculated + // The clearCacheDaily and refreshProblemDates methods are no longer needed as dates are + // calculated // @Scheduled(cron = "0 0 0 * * *") // Runs every day at midnight UTC // @CacheEvict(value = "problemDates", allEntries = true) // public void clearCacheDaily() { @@ -60,6 +70,7 @@ record DateRange(LocalDate start, LocalDate end) {} // @CacheEvict(value = "problemDates", allEntries = true) // public void refreshProblemDates() { - // logger.info("Manually refreshing problemDates cache at {}", ZonedDateTime.now(ZoneOffset.UTC)); + // logger.info("Manually refreshing problemDates cache at {}", + // ZonedDateTime.now(ZoneOffset.UTC)); // } } diff --git a/src/main/java/ca/gc/tbs/service/UserService.java b/src/main/java/ca/gc/tbs/service/UserService.java index f14734b2..b44a0322 100644 --- a/src/main/java/ca/gc/tbs/service/UserService.java +++ b/src/main/java/ca/gc/tbs/service/UserService.java @@ -5,8 +5,10 @@ import ca.gc.tbs.domain.User; import ca.gc.tbs.repository.RoleRepository; import ca.gc.tbs.repository.UserRepository; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.*; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.security.core.Authentication; import org.springframework.security.core.GrantedAuthority; import org.springframework.security.core.authority.SimpleGrantedAuthority; import org.springframework.security.core.context.SecurityContextHolder; @@ -16,147 +18,144 @@ import org.springframework.security.crypto.password.PasswordEncoder; import org.springframework.stereotype.Service; -import java.time.LocalDate; -import java.time.format.DateTimeFormatter; -import java.util.*; - @Service public class UserService implements UserDetailsService { - public static final String USER_ROLE = "USER"; - public static final String ADMIN_ROLE = "ADMIN"; - public static final String API_ROLE = "API"; - - public static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - - private final UserRepository userRepository; - private final RoleRepository roleRepository; - private final PasswordEncoder bCryptPasswordEncoder; - - public UserService( - UserRepository userRepository, - RoleRepository roleRepository, - @Autowired(required = false) PasswordEncoder bCryptPasswordEncoder) { - this.userRepository = userRepository; - this.roleRepository = roleRepository; - this.bCryptPasswordEncoder = bCryptPasswordEncoder; - } - - public User findUserByEmail(String email) { - return userRepository.findByEmail(email); - } - - public User findUserById(String Id) { - return userRepository.findById(Id).get(); - } - - public List findUserByRole(String role) { - var oRole = this.roleRepository.findByRole(role); - return userRepository.findByRolesContaining(oRole); - } - - public void deleteUserById(String Id) { - userRepository.deleteById(Id); - } - - public List findAllUsers() { - return userRepository.findAll(); - } - - public List findInstitutions() { - return userRepository.findAllInstitutions(); - } - - public User getCurrentUser() { - var auth = SecurityContextHolder.getContext().getAuthentication(); - if (auth.getPrincipal() instanceof org.springframework.security.core.userdetails.User springUser) { - return this.findUserByEmail(springUser.getUsername()); - } - return null; - } - - public void saveUser(User user) { - user.setPassword(bCryptPasswordEncoder.encode(user.getPassword())); - user.setDateCreated(DATE_FORMATTER.format(LocalDate.now())); - Role userRole = null; - if (this.userRepository.count() <= 0) { - user.setEnabled(true); - userRole = roleRepository.findByRole(ADMIN_ROLE); - } else { - userRole = roleRepository.findByRole(USER_ROLE); - } - user.setRoles(new HashSet<>(Arrays.asList(userRole))); - userRepository.save(user); - } - - public void saveApiUser(User user) { - user.setPassword(bCryptPasswordEncoder.encode(user.getPassword())); - user.setDateCreated(DATE_FORMATTER.format(LocalDate.now())); - var apiRole = roleRepository.findByRole(API_ROLE); - user.setRoles(new HashSet<>(Arrays.asList(apiRole))); - userRepository.save(user); - } - - public Role findRoleByName(String roleName) { - return roleRepository.findByRole(roleName); - } - - public boolean isAdmin(User user) { - for (Role role : user.getRoles()) { - if (role.getRole().contentEquals(ADMIN_ROLE)) { - return true; - } - } - return false; - } - - public boolean isAPI(User user) { - for (Role role : user.getRoles()) { - if (role.getRole().contentEquals(API_ROLE)) { - return true; - } - } - return false; - } - - public void enable(String id) { - var user = this.findUserById(id); - user.setEnabled(true); - userRepository.save(user); - } - - public void enableAdmin(String email) { - var user = this.findUserByEmail(email); - user.setRoles(new HashSet<>(Arrays.asList(roleRepository.findByRole(ADMIN_ROLE)))); - user.setEnabled(true); - userRepository.save(user); - } - - @Override - public UserDetails loadUserByUsername(String email) throws UsernameNotFoundException { - - var user = userRepository.findByEmail(email); - if (user != null && user.isEnabled()) { - var authorities = getUserAuthority(user.getRoles()); - return buildUserForAuthentication(user, authorities); - } else { - throw new UsernameNotFoundException("username not found"); - } - } - - private List getUserAuthority(Set userRoles) { - var roles = new HashSet(); - userRoles.forEach( - (role) -> { - roles.add(new SimpleGrantedAuthority(role.getRole())); - }); - - var grantedAuthorities = new ArrayList<>(roles); - return grantedAuthorities; - } - - private UserDetails buildUserForAuthentication(User user, List authorities) { - return new org.springframework.security.core.userdetails.User( - user.getEmail(), user.getPassword(), authorities); - } + public static final String USER_ROLE = "USER"; + public static final String ADMIN_ROLE = "ADMIN"; + public static final String API_ROLE = "API"; + + public static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + + private final UserRepository userRepository; + private final RoleRepository roleRepository; + private final PasswordEncoder bCryptPasswordEncoder; + + public UserService( + UserRepository userRepository, + RoleRepository roleRepository, + @Autowired(required = false) PasswordEncoder bCryptPasswordEncoder) { + this.userRepository = userRepository; + this.roleRepository = roleRepository; + this.bCryptPasswordEncoder = bCryptPasswordEncoder; + } + + public User findUserByEmail(String email) { + return userRepository.findByEmail(email); + } + + public User findUserById(String Id) { + return userRepository.findById(Id).get(); + } + + public List findUserByRole(String role) { + var oRole = this.roleRepository.findByRole(role); + return userRepository.findByRolesContaining(oRole); + } + + public void deleteUserById(String Id) { + userRepository.deleteById(Id); + } + + public List findAllUsers() { + return userRepository.findAll(); + } + + public List findInstitutions() { + return userRepository.findAllInstitutions(); + } + + public User getCurrentUser() { + var auth = SecurityContextHolder.getContext().getAuthentication(); + if (auth.getPrincipal() + instanceof org.springframework.security.core.userdetails.User springUser) { + return this.findUserByEmail(springUser.getUsername()); + } + return null; + } + + public void saveUser(User user) { + user.setPassword(bCryptPasswordEncoder.encode(user.getPassword())); + user.setDateCreated(DATE_FORMATTER.format(LocalDate.now())); + Role userRole = null; + if (this.userRepository.count() <= 0) { + user.setEnabled(true); + userRole = roleRepository.findByRole(ADMIN_ROLE); + } else { + userRole = roleRepository.findByRole(USER_ROLE); + } + user.setRoles(new HashSet<>(Arrays.asList(userRole))); + userRepository.save(user); + } + + public void saveApiUser(User user) { + user.setPassword(bCryptPasswordEncoder.encode(user.getPassword())); + user.setDateCreated(DATE_FORMATTER.format(LocalDate.now())); + var apiRole = roleRepository.findByRole(API_ROLE); + user.setRoles(new HashSet<>(Arrays.asList(apiRole))); + userRepository.save(user); + } + + public Role findRoleByName(String roleName) { + return roleRepository.findByRole(roleName); + } + + public boolean isAdmin(User user) { + for (Role role : user.getRoles()) { + if (role.getRole().contentEquals(ADMIN_ROLE)) { + return true; + } + } + return false; + } + + public boolean isAPI(User user) { + for (Role role : user.getRoles()) { + if (role.getRole().contentEquals(API_ROLE)) { + return true; + } + } + return false; + } + + public void enable(String id) { + var user = this.findUserById(id); + user.setEnabled(true); + userRepository.save(user); + } + + public void enableAdmin(String email) { + var user = this.findUserByEmail(email); + user.setRoles(new HashSet<>(Arrays.asList(roleRepository.findByRole(ADMIN_ROLE)))); + user.setEnabled(true); + userRepository.save(user); + } + + @Override + public UserDetails loadUserByUsername(String email) throws UsernameNotFoundException { + + var user = userRepository.findByEmail(email); + if (user != null && user.isEnabled()) { + var authorities = getUserAuthority(user.getRoles()); + return buildUserForAuthentication(user, authorities); + } else { + throw new UsernameNotFoundException("username not found"); + } + } + + private List getUserAuthority(Set userRoles) { + var roles = new HashSet(); + userRoles.forEach( + (role) -> { + roles.add(new SimpleGrantedAuthority(role.getRole())); + }); + + var grantedAuthorities = new ArrayList<>(roles); + return grantedAuthorities; + } + + private UserDetails buildUserForAuthentication(User user, List authorities) { + return new org.springframework.security.core.userdetails.User( + user.getEmail(), user.getPassword(), authorities); + } } diff --git a/src/main/java/org/springframework/data/mongodb/datatables/DataTablesInput.java b/src/main/java/org/springframework/data/mongodb/datatables/DataTablesInput.java index a75886a8..6c1f482b 100644 --- a/src/main/java/org/springframework/data/mongodb/datatables/DataTablesInput.java +++ b/src/main/java/org/springframework/data/mongodb/datatables/DataTablesInput.java @@ -4,8 +4,8 @@ import java.util.List; /** - * Request parameters sent by jQuery DataTables. - * Maps the server-side processing protocol from DataTables. + * Request parameters sent by jQuery DataTables. Maps the server-side processing protocol from + * DataTables. */ public class DataTablesInput { diff --git a/src/main/java/org/springframework/data/mongodb/datatables/DataTablesOutput.java b/src/main/java/org/springframework/data/mongodb/datatables/DataTablesOutput.java index f0c46d20..7291e422 100644 --- a/src/main/java/org/springframework/data/mongodb/datatables/DataTablesOutput.java +++ b/src/main/java/org/springframework/data/mongodb/datatables/DataTablesOutput.java @@ -5,8 +5,8 @@ import java.util.List; /** - * Response object for jQuery DataTables server-side processing. - * Contains the data plus pagination metadata. + * Response object for jQuery DataTables server-side processing. Contains the data plus pagination + * metadata. */ public class DataTablesOutput { diff --git a/src/main/java/org/springframework/data/mongodb/datatables/DataTablesRepository.java b/src/main/java/org/springframework/data/mongodb/datatables/DataTablesRepository.java index 9ecf1995..50e3b8ed 100644 --- a/src/main/java/org/springframework/data/mongodb/datatables/DataTablesRepository.java +++ b/src/main/java/org/springframework/data/mongodb/datatables/DataTablesRepository.java @@ -5,32 +5,29 @@ import org.springframework.data.repository.NoRepositoryBean; /** - * Extension of MongoRepository that adds DataTables server-side processing support. - * Provides findAll methods that accept DataTablesInput and return DataTablesOutput. - * Compatible with Azure CosmosDB MongoDB API. + * Extension of MongoRepository that adds DataTables server-side processing support. Provides + * findAll methods that accept DataTablesInput and return DataTablesOutput. Compatible with Azure + * CosmosDB MongoDB API. */ @NoRepositoryBean public interface DataTablesRepository extends MongoRepository { - /** - * Find all entities matching the DataTables request parameters. - */ + /** Find all entities matching the DataTables request parameters. */ DataTablesOutput findAll(DataTablesInput input); /** - * Find all entities matching the DataTables request parameters - * with additional MongoDB criteria. + * Find all entities matching the DataTables request parameters with additional MongoDB criteria. */ DataTablesOutput findAll(DataTablesInput input, Criteria additionalCriteria); /** - * Find all entities matching the DataTables request parameters - * with additional MongoDB criteria and a pre-computed total count. - * This avoids expensive count queries on large collections. + * Find all entities matching the DataTables request parameters with additional MongoDB criteria + * and a pre-computed total count. This avoids expensive count queries on large collections. * * @param input DataTables input parameters * @param additionalCriteria Additional MongoDB criteria * @param cachedTotalCount Pre-computed total record count (use -1 to query) */ - DataTablesOutput findAll(DataTablesInput input, Criteria additionalCriteria, long cachedTotalCount); + DataTablesOutput findAll( + DataTablesInput input, Criteria additionalCriteria, long cachedTotalCount); } diff --git a/src/main/java/org/springframework/data/mongodb/datatables/DataTablesRepositoryFactoryBean.java b/src/main/java/org/springframework/data/mongodb/datatables/DataTablesRepositoryFactoryBean.java index df1f9603..4e37814e 100644 --- a/src/main/java/org/springframework/data/mongodb/datatables/DataTablesRepositoryFactoryBean.java +++ b/src/main/java/org/springframework/data/mongodb/datatables/DataTablesRepositoryFactoryBean.java @@ -1,5 +1,6 @@ package org.springframework.data.mongodb.datatables; +import java.io.Serializable; import org.springframework.data.mongodb.core.MongoOperations; import org.springframework.data.mongodb.repository.query.MongoEntityInformation; import org.springframework.data.mongodb.repository.support.MongoRepositoryFactory; @@ -9,13 +10,12 @@ import org.springframework.data.repository.core.RepositoryMetadata; import org.springframework.data.repository.core.support.RepositoryFactorySupport; -import java.io.Serializable; - /** - * Factory bean that creates DataTablesRepository instances. - * Register via @EnableMongoRepositories(repositoryFactoryBeanClass = DataTablesRepositoryFactoryBean.class) + * Factory bean that creates DataTablesRepository instances. Register + * via @EnableMongoRepositories(repositoryFactoryBeanClass = DataTablesRepositoryFactoryBean.class) */ -public class DataTablesRepositoryFactoryBean, T, ID extends Serializable> +public class DataTablesRepositoryFactoryBean< + R extends Repository, T, ID extends Serializable> extends MongoRepositoryFactoryBean { public DataTablesRepositoryFactoryBean(Class repositoryInterface) { diff --git a/src/main/java/org/springframework/data/mongodb/datatables/DataTablesRepositoryImpl.java b/src/main/java/org/springframework/data/mongodb/datatables/DataTablesRepositoryImpl.java index 59d76a20..90ec1b07 100644 --- a/src/main/java/org/springframework/data/mongodb/datatables/DataTablesRepositoryImpl.java +++ b/src/main/java/org/springframework/data/mongodb/datatables/DataTablesRepositoryImpl.java @@ -3,7 +3,6 @@ import java.util.ArrayList; import java.util.List; import java.util.regex.Pattern; - import org.bson.Document; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -17,17 +16,15 @@ import org.springframework.data.mongodb.repository.support.SimpleMongoRepository; /** - * Implementation of DataTablesRepository that builds MongoDB queries - * from DataTables request parameters. + * Implementation of DataTablesRepository that builds MongoDB queries from DataTables request + * parameters. * - * Designed for Azure CosmosDB compatibility: - * - Uses $regex instead of $text for search (CosmosDB doesn't support text indexes) - * - Uses estimatedDocumentCount() for unfiltered totals (fast, metadata-based) - * - Uses $match + $count aggregation for filtered counts (CosmosDB compatible) - * - Uses skip/limit for pagination + *

Designed for Azure CosmosDB compatibility: - Uses $regex instead of $text for search (CosmosDB + * doesn't support text indexes) - Uses estimatedDocumentCount() for unfiltered totals (fast, + * metadata-based) - Uses $match + $count aggregation for filtered counts (CosmosDB compatible) - + * Uses skip/limit for pagination */ -public class DataTablesRepositoryImpl - extends SimpleMongoRepository +public class DataTablesRepositoryImpl extends SimpleMongoRepository implements DataTablesRepository { private static final Logger LOG = LoggerFactory.getLogger(DataTablesRepositoryImpl.class); @@ -53,7 +50,8 @@ public DataTablesOutput findAll(DataTablesInput input, Criteria additionalCri } @Override - public DataTablesOutput findAll(DataTablesInput input, Criteria additionalCriteria, long cachedTotalCount) { + public DataTablesOutput findAll( + DataTablesInput input, Criteria additionalCriteria, long cachedTotalCount) { long methodStart = System.currentTimeMillis(); DataTablesOutput output = new DataTablesOutput<>(); output.setDraw(input.getDraw()); @@ -84,7 +82,10 @@ public DataTablesOutput findAll(DataTablesInput input, Criteria additionalCri LOG.debug("PERF: Using cached total count: {}", totalCount); } else { totalCount = countDocuments(finalCriteria, collectionName); - LOG.debug("PERF: Count took {}ms (result: {})", System.currentTimeMillis() - methodStart, totalCount); + LOG.debug( + "PERF: Count took {}ms (result: {})", + System.currentTimeMillis() - methodStart, + totalCount); } output.setRecordsTotal(totalCount); output.setRecordsFiltered(totalCount); @@ -104,7 +105,10 @@ public DataTablesOutput findAll(DataTablesInput input, Criteria additionalCri long dataQueryStart = System.currentTimeMillis(); List data = mongoOperations.find(query, entityClass, collectionName); output.setData(data); - LOG.debug("PERF: Data query took {}ms (returned {} records)", System.currentTimeMillis() - dataQueryStart, data.size()); + LOG.debug( + "PERF: Data query took {}ms (returned {} records)", + System.currentTimeMillis() - dataQueryStart, + data.size()); } catch (Exception e) { LOG.error("Error executing DataTables query", e); @@ -117,10 +121,10 @@ public DataTablesOutput findAll(DataTablesInput input, Criteria additionalCri } /** - * Count documents matching the given criteria. - * - Empty/null criteria: uses estimatedDocumentCount() - reads collection metadata, no scan. - * - With criteria: uses $match + $count aggregation pipeline - CosmosDB compatible. - * (MongoTemplate.count() uses countDocuments() which runs $group+$sum and times out on CosmosDB) + * Count documents matching the given criteria. - Empty/null criteria: uses + * estimatedDocumentCount() - reads collection metadata, no scan. - With criteria: uses $match + + * $count aggregation pipeline - CosmosDB compatible. (MongoTemplate.count() uses countDocuments() + * which runs $group+$sum and times out on CosmosDB) */ private long countDocuments(Criteria criteria, String collectionName) { boolean isEmpty = criteria == null || criteria.equals(new Criteria()); @@ -129,10 +133,8 @@ private long countDocuments(Criteria criteria, String collectionName) { return mongoOperations.getCollection(collectionName).estimatedDocumentCount(); } // $match + $count aggregation is CosmosDB-compatible and avoids the $group+$sum timeout - Aggregation countAgg = Aggregation.newAggregation( - Aggregation.match(criteria), - Aggregation.count().as("n") - ); + Aggregation countAgg = + Aggregation.newAggregation(Aggregation.match(criteria), Aggregation.count().as("n")); AggregationResults results = mongoOperations.aggregate(countAgg, collectionName, Document.class); Document countDoc = results.getUniqueMappedResult(); @@ -140,8 +142,8 @@ private long countDocuments(Criteria criteria, String collectionName) { } /** - * Builds search criteria from DataTables input. - * Uses $regex for CosmosDB compatibility (no $text support). + * Builds search criteria from DataTables input. Uses $regex for CosmosDB compatibility (no $text + * support). */ private Criteria buildSearchCriteria(DataTablesInput input) { List criteriaList = new ArrayList<>(); @@ -163,8 +165,7 @@ private Criteria buildSearchCriteria(DataTablesInput input) { } if (!searchCriteria.isEmpty()) { - criteriaList.add( - new Criteria().orOperator(searchCriteria.toArray(new Criteria[0]))); + criteriaList.add(new Criteria().orOperator(searchCriteria.toArray(new Criteria[0]))); } } @@ -192,9 +193,7 @@ private Criteria buildSearchCriteria(DataTablesInput input) { return new Criteria().andOperator(criteriaList.toArray(new Criteria[0])); } - /** - * Builds sort from DataTables order specification. - */ + /** Builds sort from DataTables order specification. */ private Sort buildSort(DataTablesInput input) { List orders = new ArrayList<>(); @@ -203,9 +202,7 @@ private Sort buildSort(DataTablesInput input) { DataTablesInput.Column column = input.getColumns().get(order.getColumn()); if (column.isOrderable() && column.getData() != null && !column.getData().isEmpty()) { Sort.Direction direction = - "desc".equalsIgnoreCase(order.getDir()) - ? Sort.Direction.DESC - : Sort.Direction.ASC; + "desc".equalsIgnoreCase(order.getDir()) ? Sort.Direction.DESC : Sort.Direction.ASC; orders.add(new Sort.Order(direction, column.getData())); } } diff --git a/src/test/java/ca/gc/tbs/config/CustomizeAuthenticationSuccessHandlerTest.java b/src/test/java/ca/gc/tbs/config/CustomizeAuthenticationSuccessHandlerTest.java index d88c4c4c..a75a99ce 100644 --- a/src/test/java/ca/gc/tbs/config/CustomizeAuthenticationSuccessHandlerTest.java +++ b/src/test/java/ca/gc/tbs/config/CustomizeAuthenticationSuccessHandlerTest.java @@ -1,22 +1,19 @@ package ca.gc.tbs.config; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; +import java.util.List; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; -import org.mockito.Mockito; import org.springframework.security.core.Authentication; import org.springframework.security.core.GrantedAuthority; import org.springframework.security.web.savedrequest.RequestCache; import org.springframework.security.web.savedrequest.SavedRequest; -import java.util.Collection; -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.*; - class CustomizeAuthenticationSuccessHandlerTest { @Test diff --git a/src/test/java/ca/gc/tbs/controller/AuthControllerTest.java b/src/test/java/ca/gc/tbs/controller/AuthControllerTest.java index 01807002..e2f406d3 100644 --- a/src/test/java/ca/gc/tbs/controller/AuthControllerTest.java +++ b/src/test/java/ca/gc/tbs/controller/AuthControllerTest.java @@ -56,9 +56,10 @@ void createApiUserAcceptsPost() throws Exception { MockMvcRequestBuilders.post("/createApiUser") .contentType(MediaType.APPLICATION_JSON) .content("{\"username\":\"test@example.com\",\"password\":\"secret\"}")) - .andExpect(result -> - org.assertj.core.api.Assertions.assertThat(result.getResponse().getStatus()) - .isNotEqualTo(405)); + .andExpect( + result -> + org.assertj.core.api.Assertions.assertThat(result.getResponse().getStatus()) + .isNotEqualTo(405)); } @Configuration @@ -67,11 +68,12 @@ static class TestSecurityConfig { @Bean SecurityFilterChain filterChain(HttpSecurity http) throws Exception { - http - .csrf(csrf -> csrf.disable()) + http.csrf(csrf -> csrf.disable()) .authorizeHttpRequests(auth -> auth.anyRequest().authenticated()) - .exceptionHandling(ex -> ex.authenticationEntryPoint( - (request, response, authException) -> response.sendError(401))); + .exceptionHandling( + ex -> + ex.authenticationEntryPoint( + (request, response, authException) -> response.sendError(401))); return http.build(); } } diff --git a/src/test/java/ca/gc/tbs/controller/UserControllerSecurityTest.java b/src/test/java/ca/gc/tbs/controller/UserControllerSecurityTest.java index fd76c366..99f1b95a 100644 --- a/src/test/java/ca/gc/tbs/controller/UserControllerSecurityTest.java +++ b/src/test/java/ca/gc/tbs/controller/UserControllerSecurityTest.java @@ -1,5 +1,8 @@ package ca.gc.tbs.controller; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + import ca.gc.tbs.service.UserService; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -15,10 +18,9 @@ import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; import org.springframework.test.web.servlet.result.MockMvcResultMatchers; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; -@SpringBootTest(classes = {UserController.class, UserControllerSecurityTest.TestSecurityConfig.class}) +@SpringBootTest( + classes = {UserController.class, UserControllerSecurityTest.TestSecurityConfig.class}) @AutoConfigureMockMvc class UserControllerSecurityTest { @@ -64,11 +66,12 @@ static class TestSecurityConfig { @Bean SecurityFilterChain filterChain(HttpSecurity http) throws Exception { - http - .csrf(csrf -> csrf.disable()) + http.csrf(csrf -> csrf.disable()) .authorizeHttpRequests(auth -> auth.anyRequest().authenticated()) - .exceptionHandling(ex -> ex.authenticationEntryPoint( - (request, response, authException) -> response.sendRedirect("/login"))); + .exceptionHandling( + ex -> + ex.authenticationEntryPoint( + (request, response, authException) -> response.sendRedirect("/login"))); return http.build(); } diff --git a/src/test/java/ca/gc/tbs/controller/UserControllerXssTest.java b/src/test/java/ca/gc/tbs/controller/UserControllerXssTest.java index 06e36c12..1dbaf5e3 100644 --- a/src/test/java/ca/gc/tbs/controller/UserControllerXssTest.java +++ b/src/test/java/ca/gc/tbs/controller/UserControllerXssTest.java @@ -1,5 +1,7 @@ package ca.gc.tbs.controller; +import static org.assertj.core.api.Assertions.assertThat; + import ca.gc.tbs.domain.Role; import ca.gc.tbs.domain.User; import ca.gc.tbs.service.UserService; @@ -8,8 +10,6 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; -import static org.assertj.core.api.Assertions.assertThat; - class UserControllerXssTest { @Test diff --git a/src/test/java/ca/gc/tbs/filter/GcIpFilterTest.java b/src/test/java/ca/gc/tbs/filter/GcIpFilterTest.java index e89663d9..9c9ebf1a 100644 --- a/src/test/java/ca/gc/tbs/filter/GcIpFilterTest.java +++ b/src/test/java/ca/gc/tbs/filter/GcIpFilterTest.java @@ -1,124 +1,123 @@ package ca.gc.tbs.filter; +import static org.mockito.Mockito.*; + import ca.gc.tbs.service.GcIpValidationService; import jakarta.servlet.FilterChain; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; +import java.io.PrintWriter; +import java.io.StringWriter; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.test.util.ReflectionTestUtils; -import java.io.PrintWriter; -import java.io.StringWriter; - -import static org.mockito.Mockito.*; - class GcIpFilterTest { - private GcIpFilter filter; - private GcIpValidationService gcIpValidationService; - private HttpServletRequest request; - private HttpServletResponse response; - private FilterChain chain; + private GcIpFilter filter; + private GcIpValidationService gcIpValidationService; + private HttpServletRequest request; + private HttpServletResponse response; + private FilterChain chain; - private static final String WHITELISTED_IP = "205.193.96.10"; - private static final String NON_GC_IP = "1.2.3.4"; + private static final String WHITELISTED_IP = "205.193.96.10"; + private static final String NON_GC_IP = "1.2.3.4"; - @BeforeEach - void setUp() throws Exception { - gcIpValidationService = mock(GcIpValidationService.class); - request = mock(HttpServletRequest.class); - response = mock(HttpServletResponse.class); - chain = mock(FilterChain.class); + @BeforeEach + void setUp() throws Exception { + gcIpValidationService = mock(GcIpValidationService.class); + request = mock(HttpServletRequest.class); + response = mock(HttpServletResponse.class); + chain = mock(FilterChain.class); - filter = new GcIpFilter(); - ReflectionTestUtils.setField(filter, "gcIpValidationService", gcIpValidationService); - ReflectionTestUtils.setField(filter, "filterEnabled", true); - ReflectionTestUtils.setField(filter, "whitelistIps", WHITELISTED_IP); - ReflectionTestUtils.setField(filter, "whitelistFilePath", ""); - ReflectionTestUtils.setField(filter, "fileWhitelistIps", new java.util.HashSet<>()); + filter = new GcIpFilter(); + ReflectionTestUtils.setField(filter, "gcIpValidationService", gcIpValidationService); + ReflectionTestUtils.setField(filter, "filterEnabled", true); + ReflectionTestUtils.setField(filter, "whitelistIps", WHITELISTED_IP); + ReflectionTestUtils.setField(filter, "whitelistFilePath", ""); + ReflectionTestUtils.setField(filter, "fileWhitelistIps", new java.util.HashSet<>()); - when(request.getRequestURI()).thenReturn("/some/path"); - when(gcIpValidationService.isGcIp(anyString())).thenReturn(false); + when(request.getRequestURI()).thenReturn("/some/path"); + when(gcIpValidationService.isGcIp(anyString())).thenReturn(false); - PrintWriter writer = new PrintWriter(new StringWriter()); - when(response.getWriter()).thenReturn(writer); - } + PrintWriter writer = new PrintWriter(new StringWriter()); + when(response.getWriter()).thenReturn(writer); + } - // --- X_REAL_IP strategy --- + // --- X_REAL_IP strategy --- - @Test - void xRealIp_spoofedForwardedForIsIgnored() throws Exception { - ReflectionTestUtils.setField(filter, "clientIpSource", "X_REAL_IP"); + @Test + void xRealIp_spoofedForwardedForIsIgnored() throws Exception { + ReflectionTestUtils.setField(filter, "clientIpSource", "X_REAL_IP"); - // Attacker sends a whitelisted GC IP in X-Forwarded-For - when(request.getHeader("X-Forwarded-For")).thenReturn(WHITELISTED_IP); - // X-Real-IP is not present (nginx didn't set it, so request is direct/untrusted) - when(request.getHeader("X-Real-IP")).thenReturn(null); - when(request.getRemoteAddr()).thenReturn(NON_GC_IP); + // Attacker sends a whitelisted GC IP in X-Forwarded-For + when(request.getHeader("X-Forwarded-For")).thenReturn(WHITELISTED_IP); + // X-Real-IP is not present (nginx didn't set it, so request is direct/untrusted) + when(request.getHeader("X-Real-IP")).thenReturn(null); + when(request.getRemoteAddr()).thenReturn(NON_GC_IP); - filter.doFilter(request, response, chain); + filter.doFilter(request, response, chain); - verify(response).setStatus(HttpServletResponse.SC_FORBIDDEN); - verify(chain, never()).doFilter(any(), any()); - } + verify(response).setStatus(HttpServletResponse.SC_FORBIDDEN); + verify(chain, never()).doFilter(any(), any()); + } - @Test - void xRealIp_trustedHeaderFromNginxIsAccepted() throws Exception { - ReflectionTestUtils.setField(filter, "clientIpSource", "X_REAL_IP"); + @Test + void xRealIp_trustedHeaderFromNginxIsAccepted() throws Exception { + ReflectionTestUtils.setField(filter, "clientIpSource", "X_REAL_IP"); - when(request.getHeader("X-Real-IP")).thenReturn(WHITELISTED_IP); + when(request.getHeader("X-Real-IP")).thenReturn(WHITELISTED_IP); - filter.doFilter(request, response, chain); + filter.doFilter(request, response, chain); - verify(chain).doFilter(request, response); - verify(response, never()).setStatus(HttpServletResponse.SC_FORBIDDEN); - } + verify(chain).doFilter(request, response); + verify(response, never()).setStatus(HttpServletResponse.SC_FORBIDDEN); + } - // --- X_FORWARDED_FOR_LAST strategy --- + // --- X_FORWARDED_FOR_LAST strategy --- - @Test - void xForwardedForLast_attackerPrefixedGcIpIsIgnored() throws Exception { - ReflectionTestUtils.setField(filter, "clientIpSource", "X_FORWARDED_FOR_LAST"); + @Test + void xForwardedForLast_attackerPrefixedGcIpIsIgnored() throws Exception { + ReflectionTestUtils.setField(filter, "clientIpSource", "X_FORWARDED_FOR_LAST"); - // Attacker prepends a whitelisted GC IP; ALB appends the real (non-GC) client IP at the end - when(request.getHeader("X-Forwarded-For")).thenReturn(WHITELISTED_IP + ", " + NON_GC_IP); - when(request.getRemoteAddr()).thenReturn("10.0.0.1"); // ALB private IP + // Attacker prepends a whitelisted GC IP; ALB appends the real (non-GC) client IP at the end + when(request.getHeader("X-Forwarded-For")).thenReturn(WHITELISTED_IP + ", " + NON_GC_IP); + when(request.getRemoteAddr()).thenReturn("10.0.0.1"); // ALB private IP - filter.doFilter(request, response, chain); + filter.doFilter(request, response, chain); - verify(response).setStatus(HttpServletResponse.SC_FORBIDDEN); - verify(chain, never()).doFilter(any(), any()); - } + verify(response).setStatus(HttpServletResponse.SC_FORBIDDEN); + verify(chain, never()).doFilter(any(), any()); + } - @Test - void xForwardedForLast_realGcIpAppendedByAlbIsAccepted() throws Exception { - ReflectionTestUtils.setField(filter, "clientIpSource", "X_FORWARDED_FOR_LAST"); + @Test + void xForwardedForLast_realGcIpAppendedByAlbIsAccepted() throws Exception { + ReflectionTestUtils.setField(filter, "clientIpSource", "X_FORWARDED_FOR_LAST"); - // ALB appends the real client IP (a whitelisted GC IP) as the last entry - when(request.getHeader("X-Forwarded-For")).thenReturn("198.51.100.1, " + WHITELISTED_IP); - when(request.getRemoteAddr()).thenReturn("10.0.0.1"); + // ALB appends the real client IP (a whitelisted GC IP) as the last entry + when(request.getHeader("X-Forwarded-For")).thenReturn("198.51.100.1, " + WHITELISTED_IP); + when(request.getRemoteAddr()).thenReturn("10.0.0.1"); - filter.doFilter(request, response, chain); + filter.doFilter(request, response, chain); - verify(chain).doFilter(request, response); - verify(response, never()).setStatus(HttpServletResponse.SC_FORBIDDEN); - } + verify(chain).doFilter(request, response); + verify(response, never()).setStatus(HttpServletResponse.SC_FORBIDDEN); + } - // --- REMOTE_ADDR strategy --- + // --- REMOTE_ADDR strategy --- - @Test - void remoteAddr_nonGcAddressIsBlocked() throws Exception { - ReflectionTestUtils.setField(filter, "clientIpSource", "REMOTE_ADDR"); + @Test + void remoteAddr_nonGcAddressIsBlocked() throws Exception { + ReflectionTestUtils.setField(filter, "clientIpSource", "REMOTE_ADDR"); - // Even if headers claim a GC IP, we only trust remoteAddr - when(request.getHeader("X-Forwarded-For")).thenReturn(WHITELISTED_IP); - when(request.getHeader("X-Real-IP")).thenReturn(WHITELISTED_IP); - when(request.getRemoteAddr()).thenReturn(NON_GC_IP); + // Even if headers claim a GC IP, we only trust remoteAddr + when(request.getHeader("X-Forwarded-For")).thenReturn(WHITELISTED_IP); + when(request.getHeader("X-Real-IP")).thenReturn(WHITELISTED_IP); + when(request.getRemoteAddr()).thenReturn(NON_GC_IP); - filter.doFilter(request, response, chain); + filter.doFilter(request, response, chain); - verify(response).setStatus(HttpServletResponse.SC_FORBIDDEN); - verify(chain, never()).doFilter(any(), any()); - } + verify(response).setStatus(HttpServletResponse.SC_FORBIDDEN); + verify(chain, never()).doFilter(any(), any()); + } } From 2b5c8c611c346f967292021a1e6d31eadcfb01d6 Mon Sep 17 00:00:00 2001 From: Hamza Aburaneh Date: Thu, 16 Jul 2026 08:36:27 -0400 Subject: [PATCH 4/4] fmt: file --- kubernetes/feedback-viewer-ingress.yml.gpg | Bin 1618 -> 1618 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/kubernetes/feedback-viewer-ingress.yml.gpg b/kubernetes/feedback-viewer-ingress.yml.gpg index d5b693644b4fd39afd8390c666e09e643324b084..49a17e8fd92264af3ef593434e1129e636932de1 100644 GIT binary patch literal 1618 zcmV-Y2Ceyw4Fm}T2u0qI4lkOP)b!Ho0njxm@dulg2U^Idrbv2s%zLLp@Y9QkVDY0y z_5QM95r_xM>EpKoK?DP9lFSW)cWg)ht)0OICpU&XFz?L@x!V?XrOr8UPxqg4+Z9Zn zxR=Q@fz1*EFclVK4TUV$h-mo;gABJ7Lv083BqNLu+||?Q6_DS|t8-Tl2VIW2T_a2M z2VjKP-385d^CYA&u|aK?aOsfz5sKIE2&F$4gwj54k+WG=apw%+J0f5*dhKg%hp_Q&8+Tl?PsGM4eDnjS#ny9t4RUW2(%O0RtF#zvYeLs&{}c({Hc223zm7wrZNq4ApU{ zO0MUNL?+9IBRMzW`Iv;fiGHg|+V?ZkpO|oS`S>&;37J}>><{9Qcs+?*URF;(3{778(W{-ri}bK_^s11>Pze@-1d&hS-cS zD1(p^{WPjpki!y@#9!``2fv=+86CdzPWHu8eTZ=nCy}AJX1<_Gd_r{ z%T;{2#~Rlv=((y2nwGSo9j9gH$5NzoqSc|Sd%VH9eK7ADYrd*BYIVLR@kkYB-OGyK=i36x9T zuzPJmsMmI(sGV%tN1oTLV#Ja<9OhNs!C(yPtpX2v5)+8XUa%YwZ1g}e$1pwk%)bFb zOC_SUDCAh&DpCS;Hl6pH=YeVti^%7o}sHn!i43+to>L(x> zOoQHr=JT=rM}tBX6f2?Cgk@(>l@NR^fB)&YQB~6a=w%HkiB(AwFJ!$cHw(&#R(;`( zdG|)oSYls8DQ37^s?YJv8lQ_wLa^XKFn=NWVBqqkS;|XjD&za0A99^psubk1vU;5i zF+LTrnv8*@9QtZ!IdLTLRMiv43Q0Hd}c5|LKUZLN6c-Yn6Li(ILENhXC?F1SYtq1|rh^R4=e&eIg9ZJ@cp zz68kFcYhMyD(T(ebNUHsBB+Cj@`5|;GP)+ZJY(&iG%|4RryRDpP9!B!8A|+vGa&zG;>Z@%N zpqs*;kvY*|Mcj=e@+4u}nW<=U@~Gn^&sphQciQ7^R^wd zAqHwQn&68)EEC0X))FG>bNb}C{{DfXWn)znXgZRA60#zZ>pGPjLfSfLi%AtIEi_pJ zoCkHX+8f76{ubiM1%6C8c`lru1B#eEP3f%*$eLO0>ew1mP7W?f1w51lNVPS9QAub*hR QP6S;=n{{B=iw$1Ygmn-g`2YX_ literal 1618 zcmV-Y2Ceyw4Fm}T2=|j}K+1?Wp8eA50Vc|p8OdH#|4{pNatMK)^lWr9_o7CZ%be0f zF$Q`fCHT|HN_eU~LeS7Y#wSM4ym{8@ia#4}T7sZu;>_!u{fSmqr=c(=Po_vVqMOi;TUM_ak}HS0?@O3TVzr5zF{72Ow6?+6e_s^rjuy zPfKmy+dy?oYv<~Rn-l>%uT?(`{)2Fms+m+Kryq%@3}V+q0%CpVsK%r$w~7<C+0H-N|5@!BE7HGrw~>uN|K47J+;{_;-n|zNWj7>GI%l%+xc2P<% za(HV?%Z7OGIwwc$>>Y9Ven704+%Y;nuh*VF22!ambJ|}~dXd)a){OfVN?P1_TZ5de ze-87v5*RX_OUh1SY4`CHI<{tYC!qdJD-}nT>nnGkA6#DEaw~FTG{$l>ra=H+>|aOK zia{M`nf*8kM)W2{W3nT$RwS=DQWnlZcB+LeG-Sf@BwJTV)OzLSc!5B1sir-AeU8ol zB!juxY~|i#t%g{m9&qQamOoA(GSnjOPh(y>{olEpc6@?kpE+7W2?>La!|`mK+o0We zEi3FD2$s%bmd}cyw~CYfEHYfka*W4 zE<7!}meEE6u zi1h5tYpI1a2b_ghYnH3CGQgjgrC~W?$FHeF^*&t1Ya?WLocx{?3`%o60lGldKYxgK zF`TrXzCsP-v4-@iIsjK?p8VPzjVm5Th^VVBn_jgCNt-L2!2ci$Ry@@K8=TWM@D+A_GVRX@Vnu2;@+#M*hHdZz3<4jdyKWEJ#XhaIB+cc(ApiY zF9uekwM&V(N`sEm?15Wjoq!GN_D9^a@bSA{%KIMRNuLLsLpOarh?khFnaA6fI~gG> zw_qkb2yZ;B$CPXL#E zEGcS_rvP0yP*q_ms@#q~;c#l*9;2cpy*|KkQa^HiWgImyJ=}(@a7y-3m%K2=l`*Io zp71r*4h&MT=T^tr5`pCN-RBU}j~03VG)uw+al~1G!vv4(e4O?H1s$p3gmh;3VKLHE zGQZh5;tz)2R1Bwen|3C4xCoIecGR!o15-X#o$1n$aF1oCRLNNdK2(8*4a^`Yp6JMK zX1fI%8l%jb);;t*Rh@*D7PeBYK^Egsf-Tt;Ba_b|`w|4&7I)lolrbcIdLK70VZT`i@Ky Q+96e#Q`ugGv@mD!Msh1BJpcdz