diff --git a/package.json b/package.json index cb0a2eac6..94c524fbb 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "lint": "eslint \"./**/*.js*\"", "lint:fix": "eslint --fix \"./**/*.js*\"", "test": "vitest", - "check-env": "node ./utils/checkEnv" + "check-env": "node ./src/utils/checkEnv" }, "husky": { "hooks": { diff --git a/src/scripts/updateHfDataset.py b/src/scripts/updateHfDataset.py index c8d505c36..b7b7a4534 100644 --- a/src/scripts/updateHfDataset.py +++ b/src/scripts/updateHfDataset.py @@ -5,8 +5,12 @@ from tqdm import tqdm from huggingface_hub import HfApi, login from dotenv import load_dotenv +from datetime import datetime load_dotenv() +def get_current_year(): + return str(datetime.now().year) + # set environment as 'dev' or 'prod' ENV = os.getenv('VITE_ENV') @@ -21,26 +25,35 @@ def dlData(): ''' - Download the current year's dataset from data.lacity.org + Download the current year's dataset from data.lacity.org. + Returns the year, so it can be passed to subsequent steps. ''' + year = get_current_year() url = "https://data.lacity.org/api/views/h73f-gn57/rows.csv?accessType=DOWNLOAD" - outfile = "2025.csv" + outfile = f"{year}.csv" response = requests.get(url, stream=True) + # If we get a 4xx or 5xx HTTP status, raise an exception and stop processing altogether + response.raise_for_status() + # Save downloaded file with open(outfile, "wb") as file: for data in tqdm(response.iter_content()): file.write(data) + return year + -def hfClean(): +def hfClean(year=None): ''' Clean the dataset by removing problematic string combinations and update timestamp to ISO format ''' - infile = "2025.csv" - fixed_filename = "2025-fixed.csv" - clean_filename = "2025-clean.parquet" + if year is None: + year = get_current_year() + infile = f"{year}.csv" + fixed_filename = f"{year}-fixed.csv" + clean_filename = f"{year}-clean.parquet" # List of problmenatic strings to be replaced with "" replace_strings = ["VE, 0"] @@ -65,13 +78,15 @@ def hfClean(): print(f"File {infile} not found.") -def hfUpload(): +def hfUpload(year=None): ''' Upload the clean dataset to huggingface.co ''' - local_filename = '2025-clean.parquet' - dest_filename = '2025.parquet' - repo_name = '2025' + if year is None: + year = get_current_year() + local_filename = f'{year}-clean.parquet' + dest_filename = f'{year}.parquet' + repo_name = f'{year}' repo_type = 'dataset' repo_id = f"{HF_USERNAME}/{repo_name}" @@ -95,9 +110,9 @@ def cleanUp(): def main(): - dlData() - hfClean() - hfUpload() + year = dlData() + hfClean(year) + hfUpload(year) cleanUp() diff --git a/src/utils/checkEnv.js b/src/utils/checkEnv.js index e9fde3b43..7781c3c51 100644 --- a/src/utils/checkEnv.js +++ b/src/utils/checkEnv.js @@ -13,8 +13,8 @@ const red = '\x1b[31m%s\x1b[0m'; const green = '\x1b[32m%s\x1b[0m'; const vitePrefix = 'VITE_'; -const envPath = path.resolve(__dirname, '../.env'); -const exampleEnvPath = path.resolve(__dirname, '../.example.env'); +const envPath = path.resolve(__dirname, '../../.env'); +const exampleEnvPath = path.resolve(__dirname, '../../.example.env'); function getEnv(fileName) { return dotenv.parse(fs.readFileSync(fileName)); diff --git a/src/utils/test-utils.jsx b/src/utils/test-utils.jsx index 2b69981e2..8cc681431 100644 --- a/src/utils/test-utils.jsx +++ b/src/utils/test-utils.jsx @@ -1,7 +1,9 @@ import React from 'react' import { render } from '@testing-library/react' import { Provider } from 'react-redux' +import { ThemeProvider } from '@mui/material/styles' import { setupStore } from '../redux/store' +import theme from '@theme/theme' // https://redux.js.org/usage/writing-tests#setting-up-a-reusable-test-render-function export function renderWithProviders(ui, extendedRenderOptions = {}) { @@ -13,7 +15,9 @@ export function renderWithProviders(ui, extendedRenderOptions = {}) { } = extendedRenderOptions const Wrapper = ({ children }) => ( - {children} + + {children} + ) // Return an object with the store and all of RTL's query functions diff --git a/vite.config.js b/vite.config.js index 37111b246..e8807ef2b 100644 --- a/vite.config.js +++ b/vite.config.js @@ -31,7 +31,7 @@ export default defineConfig(() => { }, test: { environment: 'jsdom', - setupFiles: 'utils/test-setup.js' + setupFiles: 'src/utils/test-setup.js' } }; });