diff --git a/package.json b/package.json
index cb0a2eac6..94c524fbb 100644
--- a/package.json
+++ b/package.json
@@ -17,7 +17,7 @@
"lint": "eslint \"./**/*.js*\"",
"lint:fix": "eslint --fix \"./**/*.js*\"",
"test": "vitest",
- "check-env": "node ./utils/checkEnv"
+ "check-env": "node ./src/utils/checkEnv"
},
"husky": {
"hooks": {
diff --git a/src/scripts/updateHfDataset.py b/src/scripts/updateHfDataset.py
index c8d505c36..b7b7a4534 100644
--- a/src/scripts/updateHfDataset.py
+++ b/src/scripts/updateHfDataset.py
@@ -5,8 +5,12 @@
from tqdm import tqdm
from huggingface_hub import HfApi, login
from dotenv import load_dotenv
+from datetime import datetime
load_dotenv()
+def get_current_year():
+ return str(datetime.now().year)
+
# set environment as 'dev' or 'prod'
ENV = os.getenv('VITE_ENV')
@@ -21,26 +25,35 @@
def dlData():
'''
- Download the current year's dataset from data.lacity.org
+ Download the current year's dataset from data.lacity.org.
+ Returns the year, so it can be passed to subsequent steps.
'''
+ year = get_current_year()
url = "https://data.lacity.org/api/views/h73f-gn57/rows.csv?accessType=DOWNLOAD"
- outfile = "2025.csv"
+ outfile = f"{year}.csv"
response = requests.get(url, stream=True)
+ # If we get a 4xx or 5xx HTTP status, raise an exception and stop processing altogether
+ response.raise_for_status()
+
# Save downloaded file
with open(outfile, "wb") as file:
for data in tqdm(response.iter_content()):
file.write(data)
+ return year
+
-def hfClean():
+def hfClean(year=None):
'''
Clean the dataset by removing problematic string combinations and update timestamp to ISO format
'''
- infile = "2025.csv"
- fixed_filename = "2025-fixed.csv"
- clean_filename = "2025-clean.parquet"
+ if year is None:
+ year = get_current_year()
+ infile = f"{year}.csv"
+ fixed_filename = f"{year}-fixed.csv"
+ clean_filename = f"{year}-clean.parquet"
# List of problmenatic strings to be replaced with ""
replace_strings = ["VE, 0"]
@@ -65,13 +78,15 @@ def hfClean():
print(f"File {infile} not found.")
-def hfUpload():
+def hfUpload(year=None):
'''
Upload the clean dataset to huggingface.co
'''
- local_filename = '2025-clean.parquet'
- dest_filename = '2025.parquet'
- repo_name = '2025'
+ if year is None:
+ year = get_current_year()
+ local_filename = f'{year}-clean.parquet'
+ dest_filename = f'{year}.parquet'
+ repo_name = f'{year}'
repo_type = 'dataset'
repo_id = f"{HF_USERNAME}/{repo_name}"
@@ -95,9 +110,9 @@ def cleanUp():
def main():
- dlData()
- hfClean()
- hfUpload()
+ year = dlData()
+ hfClean(year)
+ hfUpload(year)
cleanUp()
diff --git a/src/utils/checkEnv.js b/src/utils/checkEnv.js
index e9fde3b43..7781c3c51 100644
--- a/src/utils/checkEnv.js
+++ b/src/utils/checkEnv.js
@@ -13,8 +13,8 @@ const red = '\x1b[31m%s\x1b[0m';
const green = '\x1b[32m%s\x1b[0m';
const vitePrefix = 'VITE_';
-const envPath = path.resolve(__dirname, '../.env');
-const exampleEnvPath = path.resolve(__dirname, '../.example.env');
+const envPath = path.resolve(__dirname, '../../.env');
+const exampleEnvPath = path.resolve(__dirname, '../../.example.env');
function getEnv(fileName) {
return dotenv.parse(fs.readFileSync(fileName));
diff --git a/src/utils/test-utils.jsx b/src/utils/test-utils.jsx
index 2b69981e2..8cc681431 100644
--- a/src/utils/test-utils.jsx
+++ b/src/utils/test-utils.jsx
@@ -1,7 +1,9 @@
import React from 'react'
import { render } from '@testing-library/react'
import { Provider } from 'react-redux'
+import { ThemeProvider } from '@mui/material/styles'
import { setupStore } from '../redux/store'
+import theme from '@theme/theme'
// https://redux.js.org/usage/writing-tests#setting-up-a-reusable-test-render-function
export function renderWithProviders(ui, extendedRenderOptions = {}) {
@@ -13,7 +15,9 @@ export function renderWithProviders(ui, extendedRenderOptions = {}) {
} = extendedRenderOptions
const Wrapper = ({ children }) => (
- {children}
+
+ {children}
+
)
// Return an object with the store and all of RTL's query functions
diff --git a/vite.config.js b/vite.config.js
index 37111b246..e8807ef2b 100644
--- a/vite.config.js
+++ b/vite.config.js
@@ -31,7 +31,7 @@ export default defineConfig(() => {
},
test: {
environment: 'jsdom',
- setupFiles: 'utils/test-setup.js'
+ setupFiles: 'src/utils/test-setup.js'
}
};
});