From d550a0cf527edd700e026f794119096c9f26f878 Mon Sep 17 00:00:00 2001 From: Carlos Barros <42662020+cmbarrosb@users.noreply.github.com> Date: Thu, 2 Oct 2025 16:03:07 +0000 Subject: [PATCH 1/2] homework part 1 --- .../data/CB_food_cleaned.csv | 105 +++ .../exercise/carlos-barros-week-02.ipynb | 717 ++++++++++++++++++ 2 files changed, 822 insertions(+) create mode 100644 Week-02-Pandas-Part-2-and-DS-Overview/data/CB_food_cleaned.csv create mode 100644 Week-02-Pandas-Part-2-and-DS-Overview/exercise/carlos-barros-week-02.ipynb diff --git a/Week-02-Pandas-Part-2-and-DS-Overview/data/CB_food_cleaned.csv b/Week-02-Pandas-Part-2-and-DS-Overview/data/CB_food_cleaned.csv new file mode 100644 index 00000000..08527371 --- /dev/null +++ b/Week-02-Pandas-Part-2-and-DS-Overview/data/CB_food_cleaned.csv @@ -0,0 +1,105 @@ +Gender,calories_day,weight +1,3.0,155.0 +1,4.0, +1,3.0, +1,2.0,190.0 +1,3.0,190.0 +2,3.0,180.0 +1,3.0,137.0 +1,3.0,125.0 +1,3.0,116.0 +1,4.0,110.0 +2,3.0,264.0 +1,3.0,123.0 +2,3.0,185.0 +1,3.0,145.0 +2,3.0,170.0 +1,3.0,135.0 +2,2.0,165.0 +2,3.0,175.0 +2,3.0,195.0 +2,3.0,185.0 +2,3.0,185.0 +1,2.0,105.0 +1,3.0,125.0 +2,2.0,160.0 +2,4.0,175.0 +2,2.0,180.0 +2,2.0,167.0 +1,3.0,115.0 +2,3.0,205.0 +1,3.0,128.0 +1,3.0,150.0 +1,2.0,150.0 +1,3.0,150.0 +1,4.0,170.0 +1,3.0,150.0 +2,3.0,140.0 +1,4.0,120.0 +1,3.0,135.0 +1,2.0,100.0 +1,4.0,170.0 +1,3.0,113.0 +2,2.0,168.0 +2,3.0,150.0 +2,3.0,169.0 +2,4.0,185.0 +2,4.0,200.0 +2,3.0,165.0 +1,2.0,192.0 +2,4.0,175.0 +1,4.0,140.0 +1,3.0,155.0 +1,4.0,135.0 +1,2.0,118.0 +2,4.0,210.0 +1,4.0,180.0 +1,3.0,140.0 +1,3.0,125.0 +1,2.0, +1,3.0,145.0 +1,4.0,130.0 +1,3.0,140.0 +2,3.0,140.0 +2,4.0,200.0 +1,3.0,120.0 +1,3.0,150.0 +2,2.0,200.0 +1,3.0,135.0 +2,3.0,145.0 +1,2.0,130.0 +1,3.0,190.0 +1,3.0,127.0 +1,3.0,167.0 +1,3.0,140.0 +1,3.0,190.0 +2,3.0,155.0 +2,4.0,175.0 +1,3.0,129.0 +2,4.0,260.0 +1,2.0,135.0 +2,3.0,175.0 +2,3.0,210.0 +1,3.0,155.0 +2,3.0,185.0 +1,4.0,165.0 +1,3.0,125.0 +1,4.0,135.0 +1,3.0,130.0 +1,3.0,230.0 +1,3.0,125.0 +1,3.0,130.0 +1,3.0,165.0 +1,2.0,128.0 +1,3.0,200.0 +1,3.0,160.0 +2,2.0,170.0 +1,4.0,129.0 +1,2.0,170.0 +2,3.0,138.0 +2,4.0,150.0 +1,3.0,140.0 +2,3.0,185.0 +1,4.0,156.0 +1,2.0,180.0 +2,4.0,135.0 diff --git a/Week-02-Pandas-Part-2-and-DS-Overview/exercise/carlos-barros-week-02.ipynb b/Week-02-Pandas-Part-2-and-DS-Overview/exercise/carlos-barros-week-02.ipynb new file mode 100644 index 00000000..2100d89f --- /dev/null +++ b/Week-02-Pandas-Part-2-and-DS-Overview/exercise/carlos-barros-week-02.ipynb @@ -0,0 +1,717 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Fall 2024 Data Science Track: Week 2 - Data Cleaning Exercise" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Packages, Packages, Packages!\n", + "\n", + "Import *all* the things here! You need the standard stuff: `pandas` and `numpy`.\n", + "\n", + "If you got more stuff you want to use, add them here too. 🙂" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "# Import here.\n", + "import pandas as pd\n", + "import numpy \n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Introduction\n", + "\n", + "With the packages out of the way, now you will be working with the following data sets:\n", + "\n", + "* `food_coded.csv`: [Food choices](https://www.kaggle.com/datasets/borapajo/food-choices?select=food_coded.csv) from Kaggle\n", + "* `Ask A Manager Salary Survey 2021 (Responses) - Form Responses 1.tsv`: [Ask A Manager Salary Survey 2021 (Responses)](https://docs.google.com/spreadsheets/d/1IPS5dBSGtwYVbjsfbaMCYIWnOuRmJcbequohNxCyGVw/view?&gid=1625408792) as *Tab Separated Values (.tsv)* from Google Docs\n", + "\n", + "Each one poses different challenges. But you’ll―of course―overcome them with what you learned in class! 😉" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Food Choices Data Set" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Load the Data" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "# Load the Food choices data set into a variable (e.g., df_food).\n", + "\n", + "food_data_set_path = '../data/food_coded.csv'\n", + "\n", + "\n", + "df_food = pd.read_csv(food_data_set_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Explore the Data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "How much data did you just load?" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "125" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Count by hand. (lol kidding)\n", + "len(df_food)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "What are the columns and their types in this data set?" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "GPA object\n", + "Gender int64\n", + "breakfast int64\n", + "calories_chicken int64\n", + "calories_day float64\n", + " ... \n", + "type_sports object\n", + "veggies_day int64\n", + "vitamins int64\n", + "waffle_calories int64\n", + "weight object\n", + "Length: 61, dtype: object" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Show the column names and their types.\n", + "df_food.dtypes\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Clean the Data\n", + "\n", + "Perhaps we’d like to know more another day, but the team is really interested in just the relationship between calories (`calories_day`) and weight. 
and maybe gender." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Can you remove the other columns?" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "# Remove ‘em.\n", + "df_clean = df_food[['Gender','calories_day','weight']]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "What about `NaN`s? How many are there?" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Gender 0\n", + "calories_day 19\n", + "weight 2\n", + "dtype: int64" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Count ‘em.\n", + "df_clean.isna().sum()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We gotta remove those `NaN`s―the entire row." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "# Drop ‘em.\n", + "df_clean=df_clean.dropna()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "But what about the weird non-numeric values in the column obviously meant for numeric data?\n", + "\n", + "Notice the data type of that column from when you got the types of all the columns?\n", + "\n", + "If only we could convert the column to a numeric type and drop the rows with invalid values. đŸ€”" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "# Fix that.\n", + "df_clean['weight'] = pd.to_numeric(df_clean['weight'], errors='coerce')\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now this data seems reasonably clean for our purposes! 😁\n", + "\n", + "Let’s save it somewhere to be shipped off to another teammate. đŸ’Ÿ" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "# Savey save!\n", + "df_clean.to_csv('../data/CB_food_cleaned.csv', index=False)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Ask a Manager Salary Survey 2021 (Responses) Data Set" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Load the Data" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "ename": "FileNotFoundError", + "evalue": "[Errno 2] No such file or directory: '../data/AskAManager_SalarySurvey2021_Responses.csv'", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mFileNotFoundError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[18]\u001b[39m\u001b[32m, line 2\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;66;03m# Load the Ask A Manager Salary Survey 2021 (Responses) data set into a variable (e.g., df_salary).\u001b[39;00m\n\u001b[32m----> \u001b[39m\u001b[32m2\u001b[39m df = \u001b[43mpd\u001b[49m\u001b[43m.\u001b[49m\u001b[43mread_csv\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43m../data/AskAManager_SalarySurvey2021_Responses.csv\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/.local/lib/python3.12/site-packages/pandas/io/parsers/readers.py:1026\u001b[39m, in \u001b[36mread_csv\u001b[39m\u001b[34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, date_format, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options, dtype_backend)\u001b[39m\n\u001b[32m 1013\u001b[39m kwds_defaults = _refine_defaults_read(\n\u001b[32m 1014\u001b[39m dialect,\n\u001b[32m 1015\u001b[39m delimiter,\n\u001b[32m (...)\u001b[39m\u001b[32m 1022\u001b[39m dtype_backend=dtype_backend,\n\u001b[32m 1023\u001b[39m )\n\u001b[32m 1024\u001b[39m kwds.update(kwds_defaults)\n\u001b[32m-> \u001b[39m\u001b[32m1026\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_read\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfilepath_or_buffer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/.local/lib/python3.12/site-packages/pandas/io/parsers/readers.py:620\u001b[39m, in \u001b[36m_read\u001b[39m\u001b[34m(filepath_or_buffer, kwds)\u001b[39m\n\u001b[32m 617\u001b[39m _validate_names(kwds.get(\u001b[33m\"\u001b[39m\u001b[33mnames\u001b[39m\u001b[33m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m))\n\u001b[32m 619\u001b[39m \u001b[38;5;66;03m# Create the parser.\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m620\u001b[39m parser = \u001b[43mTextFileReader\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfilepath_or_buffer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 622\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m chunksize \u001b[38;5;129;01mor\u001b[39;00m iterator:\n\u001b[32m 623\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m parser\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/.local/lib/python3.12/site-packages/pandas/io/parsers/readers.py:1620\u001b[39m, in \u001b[36mTextFileReader.__init__\u001b[39m\u001b[34m(self, f, engine, **kwds)\u001b[39m\n\u001b[32m 1617\u001b[39m \u001b[38;5;28mself\u001b[39m.options[\u001b[33m\"\u001b[39m\u001b[33mhas_index_names\u001b[39m\u001b[33m\"\u001b[39m] = kwds[\u001b[33m\"\u001b[39m\u001b[33mhas_index_names\u001b[39m\u001b[33m\"\u001b[39m]\n\u001b[32m 1619\u001b[39m \u001b[38;5;28mself\u001b[39m.handles: IOHandles | \u001b[38;5;28;01mNone\u001b[39;00m = \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m-> \u001b[39m\u001b[32m1620\u001b[39m \u001b[38;5;28mself\u001b[39m._engine = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_make_engine\u001b[49m\u001b[43m(\u001b[49m\u001b[43mf\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mengine\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/.local/lib/python3.12/site-packages/pandas/io/parsers/readers.py:1880\u001b[39m, in \u001b[36mTextFileReader._make_engine\u001b[39m\u001b[34m(self, f, engine)\u001b[39m\n\u001b[32m 1878\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[33m\"\u001b[39m\u001b[33mb\u001b[39m\u001b[33m\"\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m mode:\n\u001b[32m 1879\u001b[39m mode += \u001b[33m\"\u001b[39m\u001b[33mb\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m-> \u001b[39m\u001b[32m1880\u001b[39m \u001b[38;5;28mself\u001b[39m.handles = \u001b[43mget_handle\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 1881\u001b[39m \u001b[43m \u001b[49m\u001b[43mf\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1882\u001b[39m \u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1883\u001b[39m \u001b[43m \u001b[49m\u001b[43mencoding\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43moptions\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mencoding\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1884\u001b[39m \u001b[43m \u001b[49m\u001b[43mcompression\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43moptions\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mcompression\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1885\u001b[39m \u001b[43m \u001b[49m\u001b[43mmemory_map\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43moptions\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mmemory_map\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1886\u001b[39m \u001b[43m \u001b[49m\u001b[43mis_text\u001b[49m\u001b[43m=\u001b[49m\u001b[43mis_text\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1887\u001b[39m \u001b[43m \u001b[49m\u001b[43merrors\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43moptions\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mencoding_errors\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mstrict\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1888\u001b[39m \u001b[43m \u001b[49m\u001b[43mstorage_options\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43moptions\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mstorage_options\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1889\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1890\u001b[39m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mself\u001b[39m.handles \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m 1891\u001b[39m f = \u001b[38;5;28mself\u001b[39m.handles.handle\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/.local/lib/python3.12/site-packages/pandas/io/common.py:873\u001b[39m, in \u001b[36mget_handle\u001b[39m\u001b[34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001b[39m\n\u001b[32m 868\u001b[39m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(handle, \u001b[38;5;28mstr\u001b[39m):\n\u001b[32m 869\u001b[39m \u001b[38;5;66;03m# Check whether the filename is to be opened in binary mode.\u001b[39;00m\n\u001b[32m 870\u001b[39m \u001b[38;5;66;03m# Binary mode does not support 'encoding' and 'newline'.\u001b[39;00m\n\u001b[32m 871\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m ioargs.encoding \u001b[38;5;129;01mand\u001b[39;00m \u001b[33m\"\u001b[39m\u001b[33mb\u001b[39m\u001b[33m\"\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m ioargs.mode:\n\u001b[32m 872\u001b[39m \u001b[38;5;66;03m# Encoding\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m873\u001b[39m handle = \u001b[38;5;28;43mopen\u001b[39;49m\u001b[43m(\u001b[49m\n\u001b[32m 874\u001b[39m \u001b[43m \u001b[49m\u001b[43mhandle\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 875\u001b[39m \u001b[43m \u001b[49m\u001b[43mioargs\u001b[49m\u001b[43m.\u001b[49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 876\u001b[39m \u001b[43m \u001b[49m\u001b[43mencoding\u001b[49m\u001b[43m=\u001b[49m\u001b[43mioargs\u001b[49m\u001b[43m.\u001b[49m\u001b[43mencoding\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 877\u001b[39m \u001b[43m \u001b[49m\u001b[43merrors\u001b[49m\u001b[43m=\u001b[49m\u001b[43merrors\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 878\u001b[39m \u001b[43m \u001b[49m\u001b[43mnewline\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m 879\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 880\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 881\u001b[39m \u001b[38;5;66;03m# Binary mode\u001b[39;00m\n\u001b[32m 882\u001b[39m handle = \u001b[38;5;28mopen\u001b[39m(handle, ioargs.mode)\n", + "\u001b[31mFileNotFoundError\u001b[39m: [Errno 2] No such file or directory: '../data/AskAManager_SalarySurvey2021_Responses.csv'" + ] + } + ], + "source": [ + "# Load the Ask A Manager Salary Survey 2021 (Responses) data set into a variable (e.g., df_salary).\n", + "df = pd.read_csv('../data/AskAManager_SalarySurvey2021_Responses.csv')\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Was that hard? 🙃" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### rename the file to something that is better for all systems. \n", + "* No spaces in filename (can use '_')\n", + "* all lower case" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Explore\n", + "\n", + "You know the drill." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "How much data did you just load?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Count by hand. I’m dead serious.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "What are the columns and their types?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Show the column names and their types.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Oh
 Ugh! Give these columns easier names to work with first. 🙄" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Rename ‘em.\n", + "# Non-binding suggestions: timestamp, age, industry, title, title_context, salary, additional_compensation, currency, other_currency, salary_context, country, state, city, total_yoe, field_yoe, highest_education_completed\tgender, race\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It’s a lot, and that should not have been easy. 😏" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You’re going to have a gander at the computing/tech subset first because thats *your* industry. But first, what value corresponds to that `industry`?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# List the unique industries and a count of their instances.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "That value among the top 5 is what you’re looking for innit? Filter out all the rows not in that industry and save it into a new dataframe. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Filtery filter. (Save it to a new variable, df_salary_tech.)\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Do a sanity check to make sure that the only values you kept are the one you are filtered for. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Sanity Check \n", + "\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We are very interested in salary figures. But how many dollars đŸ’” is a euro đŸ’¶ or a pound đŸ’·? That sounds like a problem for another day. đŸ« \n", + "\n", + "For now, let’s just look at U.S. dollars (`'USD'`)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Filtery filter for just the jobs that pay in USD!\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "What we really want know is how each U.S. city pays in tech. What value in `country` represents the United States of America?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# We did filter for USD, so if we do a count of each unique country in descending count order, the relevant value(s) should show up at the top.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Clean the Data\n", + "\n", + "Well, we can’t get our answers with what we currently have, so you’ll have to make some changes." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let’s not worry about anything below the first 5 values for now. Convert the top 5 to a single canonical value―say, `'US'`, which is nice and short." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Replace them all with 'US'.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Have a look at the count of each unique country again now." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Count again.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Did you notice anything interesting?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# BONUS CREDIT: resolve [most of] those anomalous cases too without exhaustively taking every variant literally into account.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "# BONUS CREDIT: if you’ve resolved it, let’s see how well you did by counting the number of instances of each unique value.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It’s looking good so far. Let’s find out the minimum, mean, and maximum (in that order) salary by state, sorted by the mean in descending order." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Find the minimum, mean, and maximum salary in USD by U.S. state.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Well, pooh! We forgot that `salary` isn’t numeric. Something wrong must be fixed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Fix it.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let’s try that again." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Try it again. Yeah!\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "That did the trick! Now let’s narrow this to data 2021 and 2022 just because (lel). *(Hint: that timestamp column may not be a temporal type right now.)*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Filter the data to within 2021, 2022, or 2023, saving the DataFrame to a new variable, and generate the summary again.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Bonus\n", + "\n", + "Clearly, we do not have enough data to produce useful figures for the level of specificity you’ve now reached. What do you notice about Delaware and West Virginia?\n", + "\n", + "Let’s back out a bit and return to `df_salary` (which was the loaded data with renamed columns but *sans* filtering)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Bonus #0\n", + "\n", + "Apply the same steps as before to `df_salary`, but do not filter for any specific industry. Do perform the other data cleaning stuff, and get to a point where you can generate the minimum, mean, and maximum by state." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Bonus #1\n", + "\n", + "This time, format the table output nicely (*$12,345.00*) without modifying the values in the `DataFrame`. That is, `df_salary` should be identical before versus after running your code.\n", + "\n", + "(*Hint: if you run into an error about `jinja2` perhaps you need to `pip install` something.*)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Bonus #2\n", + "\n", + "Filter out the non-single-states (e.g., `'California, Colorado'`) in the most elegant way possible (i.e., *not* by blacklisting all the bad values)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Bonus #3\n", + "\n", + "Show the quantiles instead of just minimum, mean, and maximum―say 0%, 5%, 25%, 50%, 75%, 95%, and 100%. Outliers may be deceiving.\n", + "\n", + "Sort by whatever interests you―like say the *50th* percentile.\n", + "\n", + "And throw in a count by state too. It would be interesting to know how many data points contribute to the figures for each state. (*Hint: your nice formatting from Bonus #1 might not work this time around.* 😜)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.1" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 7336ff8c7806f62ea9efc98876eaa472c4a26157 Mon Sep 17 00:00:00 2001 From: Carlos Barros <42662020+cmbarrosb@users.noreply.github.com> Date: Fri, 3 Oct 2025 05:08:54 +0000 Subject: [PATCH 2/2] final homework 2 --- .../exercise/carlos-barros-week-02.ipynb | 769 ++++++++++++++++-- 1 file changed, 701 insertions(+), 68 deletions(-) diff --git a/Week-02-Pandas-Part-2-and-DS-Overview/exercise/carlos-barros-week-02.ipynb b/Week-02-Pandas-Part-2-and-DS-Overview/exercise/carlos-barros-week-02.ipynb index 2100d89f..ce3878b2 100644 --- a/Week-02-Pandas-Part-2-and-DS-Overview/exercise/carlos-barros-week-02.ipynb +++ b/Week-02-Pandas-Part-2-and-DS-Overview/exercise/carlos-barros-week-02.ipynb @@ -20,7 +20,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -163,9 +163,21 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'df_food' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mNameError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[3]\u001b[39m\u001b[32m, line 2\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;66;03m# Remove ‘em.\u001b[39;00m\n\u001b[32m----> \u001b[39m\u001b[32m2\u001b[39m df_clean = \u001b[43mdf_food\u001b[49m[[\u001b[33m'\u001b[39m\u001b[33mGender\u001b[39m\u001b[33m'\u001b[39m,\u001b[33m'\u001b[39m\u001b[33mcalories_day\u001b[39m\u001b[33m'\u001b[39m,\u001b[33m'\u001b[39m\u001b[33mweight\u001b[39m\u001b[33m'\u001b[39m]]\n", + "\u001b[31mNameError\u001b[39m: name 'df_food' is not defined" + ] + } + ], "source": [ "# Remove ‘em.\n", "df_clean = df_food[['Gender','calories_day','weight']]" @@ -251,9 +263,21 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'df_clean' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mNameError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[4]\u001b[39m\u001b[32m, line 2\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;66;03m# Savey save!\u001b[39;00m\n\u001b[32m----> \u001b[39m\u001b[32m2\u001b[39m \u001b[43mdf_clean\u001b[49m.to_csv(\u001b[33m'\u001b[39m\u001b[33m../data/CB_food_cleaned.csv\u001b[39m\u001b[33m'\u001b[39m, index=\u001b[38;5;28;01mFalse\u001b[39;00m)\n", + "\u001b[31mNameError\u001b[39m: name 'df_clean' is not defined" + ] + } + ], "source": [ "# Savey save!\n", "df_clean.to_csv('../data/CB_food_cleaned.csv', index=False)\n" @@ -275,29 +299,11 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 15, "metadata": {}, - "outputs": [ - { - "ename": "FileNotFoundError", - "evalue": "[Errno 2] No such file or directory: '../data/AskAManager_SalarySurvey2021_Responses.csv'", - "output_type": "error", - "traceback": [ - "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mFileNotFoundError\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[18]\u001b[39m\u001b[32m, line 2\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;66;03m# Load the Ask A Manager Salary Survey 2021 (Responses) data set into a variable (e.g., df_salary).\u001b[39;00m\n\u001b[32m----> \u001b[39m\u001b[32m2\u001b[39m df = \u001b[43mpd\u001b[49m\u001b[43m.\u001b[49m\u001b[43mread_csv\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43m../data/AskAManager_SalarySurvey2021_Responses.csv\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.local/lib/python3.12/site-packages/pandas/io/parsers/readers.py:1026\u001b[39m, in \u001b[36mread_csv\u001b[39m\u001b[34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, date_format, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options, dtype_backend)\u001b[39m\n\u001b[32m 1013\u001b[39m kwds_defaults = _refine_defaults_read(\n\u001b[32m 1014\u001b[39m dialect,\n\u001b[32m 1015\u001b[39m delimiter,\n\u001b[32m (...)\u001b[39m\u001b[32m 1022\u001b[39m dtype_backend=dtype_backend,\n\u001b[32m 1023\u001b[39m )\n\u001b[32m 1024\u001b[39m kwds.update(kwds_defaults)\n\u001b[32m-> \u001b[39m\u001b[32m1026\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_read\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfilepath_or_buffer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.local/lib/python3.12/site-packages/pandas/io/parsers/readers.py:620\u001b[39m, in \u001b[36m_read\u001b[39m\u001b[34m(filepath_or_buffer, kwds)\u001b[39m\n\u001b[32m 617\u001b[39m _validate_names(kwds.get(\u001b[33m\"\u001b[39m\u001b[33mnames\u001b[39m\u001b[33m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m))\n\u001b[32m 619\u001b[39m \u001b[38;5;66;03m# Create the parser.\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m620\u001b[39m parser = \u001b[43mTextFileReader\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfilepath_or_buffer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 622\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m chunksize \u001b[38;5;129;01mor\u001b[39;00m iterator:\n\u001b[32m 623\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m parser\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.local/lib/python3.12/site-packages/pandas/io/parsers/readers.py:1620\u001b[39m, in \u001b[36mTextFileReader.__init__\u001b[39m\u001b[34m(self, f, engine, **kwds)\u001b[39m\n\u001b[32m 1617\u001b[39m \u001b[38;5;28mself\u001b[39m.options[\u001b[33m\"\u001b[39m\u001b[33mhas_index_names\u001b[39m\u001b[33m\"\u001b[39m] = kwds[\u001b[33m\"\u001b[39m\u001b[33mhas_index_names\u001b[39m\u001b[33m\"\u001b[39m]\n\u001b[32m 1619\u001b[39m \u001b[38;5;28mself\u001b[39m.handles: IOHandles | \u001b[38;5;28;01mNone\u001b[39;00m = \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m-> \u001b[39m\u001b[32m1620\u001b[39m \u001b[38;5;28mself\u001b[39m._engine = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_make_engine\u001b[49m\u001b[43m(\u001b[49m\u001b[43mf\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mengine\u001b[49m\u001b[43m)\u001b[49m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.local/lib/python3.12/site-packages/pandas/io/parsers/readers.py:1880\u001b[39m, in \u001b[36mTextFileReader._make_engine\u001b[39m\u001b[34m(self, f, engine)\u001b[39m\n\u001b[32m 1878\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[33m\"\u001b[39m\u001b[33mb\u001b[39m\u001b[33m\"\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m mode:\n\u001b[32m 1879\u001b[39m mode += \u001b[33m\"\u001b[39m\u001b[33mb\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m-> \u001b[39m\u001b[32m1880\u001b[39m \u001b[38;5;28mself\u001b[39m.handles = \u001b[43mget_handle\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 1881\u001b[39m \u001b[43m \u001b[49m\u001b[43mf\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1882\u001b[39m \u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1883\u001b[39m \u001b[43m \u001b[49m\u001b[43mencoding\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43moptions\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mencoding\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1884\u001b[39m \u001b[43m \u001b[49m\u001b[43mcompression\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43moptions\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mcompression\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1885\u001b[39m \u001b[43m \u001b[49m\u001b[43mmemory_map\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43moptions\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mmemory_map\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1886\u001b[39m \u001b[43m \u001b[49m\u001b[43mis_text\u001b[49m\u001b[43m=\u001b[49m\u001b[43mis_text\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1887\u001b[39m \u001b[43m \u001b[49m\u001b[43merrors\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43moptions\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mencoding_errors\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mstrict\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1888\u001b[39m \u001b[43m \u001b[49m\u001b[43mstorage_options\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43moptions\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mstorage_options\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1889\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1890\u001b[39m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mself\u001b[39m.handles \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m 1891\u001b[39m f = \u001b[38;5;28mself\u001b[39m.handles.handle\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.local/lib/python3.12/site-packages/pandas/io/common.py:873\u001b[39m, in \u001b[36mget_handle\u001b[39m\u001b[34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001b[39m\n\u001b[32m 868\u001b[39m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(handle, \u001b[38;5;28mstr\u001b[39m):\n\u001b[32m 869\u001b[39m \u001b[38;5;66;03m# Check whether the filename is to be opened in binary mode.\u001b[39;00m\n\u001b[32m 870\u001b[39m \u001b[38;5;66;03m# Binary mode does not support 'encoding' and 'newline'.\u001b[39;00m\n\u001b[32m 871\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m ioargs.encoding \u001b[38;5;129;01mand\u001b[39;00m \u001b[33m\"\u001b[39m\u001b[33mb\u001b[39m\u001b[33m\"\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m ioargs.mode:\n\u001b[32m 872\u001b[39m \u001b[38;5;66;03m# Encoding\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m873\u001b[39m handle = \u001b[38;5;28;43mopen\u001b[39;49m\u001b[43m(\u001b[49m\n\u001b[32m 874\u001b[39m \u001b[43m \u001b[49m\u001b[43mhandle\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 875\u001b[39m \u001b[43m \u001b[49m\u001b[43mioargs\u001b[49m\u001b[43m.\u001b[49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 876\u001b[39m \u001b[43m \u001b[49m\u001b[43mencoding\u001b[49m\u001b[43m=\u001b[49m\u001b[43mioargs\u001b[49m\u001b[43m.\u001b[49m\u001b[43mencoding\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 877\u001b[39m \u001b[43m \u001b[49m\u001b[43merrors\u001b[49m\u001b[43m=\u001b[49m\u001b[43merrors\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 878\u001b[39m \u001b[43m \u001b[49m\u001b[43mnewline\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m 879\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 880\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 881\u001b[39m \u001b[38;5;66;03m# Binary mode\u001b[39;00m\n\u001b[32m 882\u001b[39m handle = \u001b[38;5;28mopen\u001b[39m(handle, ioargs.mode)\n", - "\u001b[31mFileNotFoundError\u001b[39m: [Errno 2] No such file or directory: '../data/AskAManager_SalarySurvey2021_Responses.csv'" - ] - } - ], + "outputs": [], "source": [ - "# Load the Ask A Manager Salary Survey 2021 (Responses) data set into a variable (e.g., df_salary).\n", - "df = pd.read_csv('../data/AskAManager_SalarySurvey2021_Responses.csv')\n" + "df = pd.read_csv('../data/AskAManager_SalarySurvey2021_Responses.csv', sep='\\t', on_bad_lines='warn')" ] }, { @@ -334,11 +340,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "28062" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Count by hand. I’m dead serious.\n", + "len(df)\n", "\n" ] }, @@ -351,12 +369,41 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "Timestamp object\n", + "How old are you? object\n", + "What industry do you work in? object\n", + "Job title object\n", + "If your job title needs additional context, please clarify here: object\n", + "What is your annual salary? (You'll indicate the currency in a later question. If you are part-time or hourly, please enter an annualized equivalent -- what you would earn if you worked the job 40 hours a week, 52 weeks a year.) object\n", + "How much additional monetary compensation do you get, if any (for example, bonuses or overtime in an average year)? Please only include monetary compensation here, not the value of benefits. float64\n", + "Please indicate the currency object\n", + "If \"Other,\" please indicate the currency here: object\n", + "If your income needs additional context, please provide it here: object\n", + "What country do you work in? object\n", + "If you're in the U.S., what state do you work in? object\n", + "What city do you work in? object\n", + "How many years of professional work experience do you have overall? object\n", + "How many years of professional work experience do you have in your field? object\n", + "What is your highest level of education completed? object\n", + "What is your gender? object\n", + "What is your race? (Choose all that apply.) object\n", + "dtype: object" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Show the column names and their types.\n", - "\n" + "df.dtypes\n" ] }, { @@ -368,13 +415,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ "# Rename ‘em.\n", "# Non-binding suggestions: timestamp, age, industry, title, title_context, salary, additional_compensation, currency, other_currency, salary_context, country, state, city, total_yoe, field_yoe, highest_education_completed\tgender, race\n", - "\n" + "df.columns = ['timestamp', 'age', 'industry', 'title', 'title_context', 'salary', 'additional_compensation', 'currency', 'other_currency', 'salary_context', 'country', 'state', 'city', 'total_yoe', 'field_yoe', 'highest_education_completed', 'gender', 'race']\n" ] }, { @@ -393,12 +440,32 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "industry\n", + "Computing or Tech 4699\n", + "Education (Higher Education) 2464\n", + "Nonprofits 2419\n", + "Health care 1896\n", + "Government and Public Administration 1889\n", + " ... \n", + "Warehousing 1\n", + "Education (Early Childhood Education) 1\n", + "SAAS 1\n", + "Health and Safety 1\n", + "Aerospace Manufacturing 1\n", + "Name: count, Length: 1219, dtype: int64\n" + ] + } + ], "source": [ "# List the unique industries and a count of their instances.\n", - "\n" + "print(df['industry'].value_counts())\n" ] }, { @@ -410,12 +477,12 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "metadata": {}, "outputs": [], "source": [ "# Filtery filter. (Save it to a new variable, df_salary_tech.)\n", - "\n" + "df_salary_tech = df[df['industry'] == 'Computing or Tech']" ] }, { @@ -427,13 +494,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 34, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['Computing or Tech']\n" + ] + } + ], "source": [ "# Sanity Check \n", - "\n", - "\n" + "print(df_salary_tech['industry'].unique())\n" ] }, { @@ -447,12 +521,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 39, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['USD']\n" + ] + } + ], "source": [ "# Filtery filter for just the jobs that pay in USD!\n", - "\n" + "df_salary_tech_usd = df_salary_tech[df_salary_tech['currency'] == 'USD']\n", + "print(df_salary_tech_usd['currency'].unique())" ] }, { @@ -464,12 +547,35 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 40, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "country\n", + "United States 1576\n", + "USA 1222\n", + "US 412\n", + "U.S. 108\n", + "United States of America 90\n", + " ... \n", + "Ghana 1\n", + "Nigeria 1\n", + "ss 1\n", + "Nigeria 1\n", + "Burma 1\n", + "Name: count, Length: 76, dtype: int64" + ] + }, + "execution_count": 40, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# We did filter for USD, so if we do a count of each unique country in descending count order, the relevant value(s) should show up at the top.\n", - "\n" + "df_salary_tech_usd['country'].value_counts()\n" ] }, { @@ -490,12 +596,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 45, "metadata": {}, "outputs": [], "source": [ "# Replace them all with 'US'.\n", - "\n" + "df_salary_tech_usd.loc[:,'country'] = df_salary_tech_usd['country'].replace({\n", + " 'United States': 'US',\n", + " 'USA': 'US',\n", + " 'United States of America': 'US',\n", + " 'U.S.': 'US'\n", + "})\n" ] }, { @@ -507,12 +618,35 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 46, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "country\n", + "US 3408\n", + "United States 68\n", + "Usa 59\n", + "USA 56\n", + "usa 28\n", + " ... \n", + "Ghana 1\n", + "Nigeria 1\n", + "ss 1\n", + "Nigeria 1\n", + "Burma 1\n", + "Name: count, Length: 72, dtype: int64" + ] + }, + "execution_count": 46, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Count again.\n", - "\n" + "df_salary_tech_usd['country'].value_counts()\n" ] }, { @@ -524,23 +658,90 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 57, "metadata": {}, "outputs": [], "source": [ "# BONUS CREDIT: resolve [most of] those anomalous cases too without exhaustively taking every variant literally into account.\n", - "\n" + "\n", + "df_salary_tech_usd.loc[:,'country'] = (\n", + " df_salary_tech_usd['country']\n", + " .str.strip()\n", + " .str.replace('.', '', regex=False)\n", + " .str.title()\n", + " )" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 58, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "country\n", + "Us 3436\n", + "Usa 155\n", + "United States 108\n", + "United States Of America 11\n", + "Israel 5\n", + "Canada 5\n", + "Australia 2\n", + "United State Of America 2\n", + "Unitedstates 2\n", + "United Kingdom 2\n", + "France 2\n", + "Poland 2\n", + "Brazil 2\n", + "Singapore 2\n", + "Spain 2\n", + "India 2\n", + "Unite States 2\n", + "New Zealand 2\n", + "Denmark 2\n", + "Nigeria 2\n", + "Danmark 1\n", + "Uniyed States 1\n", + "America 1\n", + "Puerto Rico 1\n", + "United State 1\n", + "Italy 1\n", + "International 1\n", + "Cuba 1\n", + "Uruguay 1\n", + "Isa 1\n", + "United Stateds 1\n", + "United Stated 1\n", + "Remote (Philippines) 1\n", + "Pakistan 1\n", + "Mexico 1\n", + "San Francisco 1\n", + "Netherlands 1\n", + "Romania 1\n", + "Japan 1\n", + "United Stares 1\n", + "China 1\n", + "Australian 1\n", + "Jamaica 1\n", + "Thailand 1\n", + "Unites States 1\n", + "Colombia 1\n", + "Ghana 1\n", + "Ss 1\n", + "Burma 1\n", + "Name: count, dtype: int64" + ] + }, + "execution_count": 58, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "\n", "# BONUS CREDIT: if you’ve resolved it, let’s see how well you did by counting the number of instances of each unique value.\n", - "\n" + "df_salary_tech_usd['country'].value_counts()\n" ] }, { @@ -554,10 +755,143 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
minmeanmax
state
Florida28800.0229567.650002600000.0
California, Oregon200000.0200000.00000200000.0
California, Colorado176000.0176000.00000176000.0
Connecticut68000.0170137.50000270000.0
California0.0155776.69375520000.0
............
MississippiNaNNaNNaN
New York, TexasNaNNaNNaN
North DakotaNaNNaNNaN
South DakotaNaNNaNNaN
Utah, VermontNaNNaNNaN
\n", + "

64 rows × 3 columns

\n", + "
" + ], + "text/plain": [ + " min mean max\n", + "state \n", + "Florida 28800.0 229567.65000 2600000.0\n", + "California, Oregon 200000.0 200000.00000 200000.0\n", + "California, Colorado 176000.0 176000.00000 176000.0\n", + "Connecticut 68000.0 170137.50000 270000.0\n", + "California 0.0 155776.69375 520000.0\n", + "... ... ... ...\n", + "Mississippi NaN NaN NaN\n", + "New York, Texas NaN NaN NaN\n", + "North Dakota NaN NaN NaN\n", + "South Dakota NaN NaN NaN\n", + "Utah, Vermont NaN NaN NaN\n", + "\n", + "[64 rows x 3 columns]" + ] + }, + "execution_count": 61, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Find the minimum, mean, and maximum salary in USD by U.S. state.\n", - "\n" + "df_us = df_salary_tech_usd[df_salary_tech_usd['country'] == 'Us']\n", + "summary = (\n", + " df_us.groupby('state')['salary']\n", + " .agg(['min', 'mean', 'max'])\n", + " .sort_values('mean', ascending=False)\n", + ")\n", + "summary" ] }, { @@ -569,11 +903,25 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 70, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_1843/1542867887.py:2: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " df_salary_tech_usd['salary'] = pd.to_numeric(df_salary_tech_usd['salary'], errors='coerce')\n" + ] + } + ], "source": [ "# Fix it.\n", + "df_salary_tech_usd['salary'] = pd.to_numeric(df_salary_tech_usd['salary'], errors='coerce')\n", "\n" ] }, @@ -586,12 +934,145 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 71, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
minmeanmax
state
Florida28800.0229567.650002600000.0
California, Oregon200000.0200000.00000200000.0
California, Colorado176000.0176000.00000176000.0
Connecticut68000.0170137.50000270000.0
California0.0155776.69375520000.0
............
MississippiNaNNaNNaN
New York, TexasNaNNaNNaN
North DakotaNaNNaNNaN
South DakotaNaNNaNNaN
Utah, VermontNaNNaNNaN
\n", + "

64 rows × 3 columns

\n", + "
" + ], + "text/plain": [ + " min mean max\n", + "state \n", + "Florida 28800.0 229567.65000 2600000.0\n", + "California, Oregon 200000.0 200000.00000 200000.0\n", + "California, Colorado 176000.0 176000.00000 176000.0\n", + "Connecticut 68000.0 170137.50000 270000.0\n", + "California 0.0 155776.69375 520000.0\n", + "... ... ... ...\n", + "Mississippi NaN NaN NaN\n", + "New York, Texas NaN NaN NaN\n", + "North Dakota NaN NaN NaN\n", + "South Dakota NaN NaN NaN\n", + "Utah, Vermont NaN NaN NaN\n", + "\n", + "[64 rows x 3 columns]" + ] + }, + "execution_count": 71, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Try it again. Yeah!\n", - "\n" + "df_us = df_salary_tech_usd[df_salary_tech_usd['country'] == 'Us']\n", + "summary = (\n", + " df_us.groupby('state')['salary']\n", + " .agg(['min', 'mean', 'max'])\n", + " .sort_values('mean', ascending=False)\n", + ")\n", + "summary\n" ] }, { @@ -603,12 +1084,164 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 73, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_1843/520620466.py:2: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " df_salary_tech_usd['timestamp'] = pd.to_datetime(df_salary_tech_usd['timestamp'], errors='coerce')\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
minmeanmax
state
California, Oregon200000.0200000.000000200000.0
California, Colorado176000.0176000.000000176000.0
Connecticut68000.0170137.500000270000.0
California0.0156316.169811520000.0
New York14000.0148213.564356590000.0
............
New York, TexasNaNNaNNaN
North DakotaNaNNaNNaN
South CarolinaNaNNaNNaN
South DakotaNaNNaNNaN
Utah, VermontNaNNaNNaN
\n", + "

64 rows × 3 columns

\n", + "
" + ], + "text/plain": [ + " min mean max\n", + "state \n", + "California, Oregon 200000.0 200000.000000 200000.0\n", + "California, Colorado 176000.0 176000.000000 176000.0\n", + "Connecticut 68000.0 170137.500000 270000.0\n", + "California 0.0 156316.169811 520000.0\n", + "New York 14000.0 148213.564356 590000.0\n", + "... ... ... ...\n", + "New York, Texas NaN NaN NaN\n", + "North Dakota NaN NaN NaN\n", + "South Carolina NaN NaN NaN\n", + "South Dakota NaN NaN NaN\n", + "Utah, Vermont NaN NaN NaN\n", + "\n", + "[64 rows x 3 columns]" + ] + }, + "execution_count": 73, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Filter the data to within 2021, 2022, or 2023, saving the DataFrame to a new variable, and generate the summary again.\n", - "\n" + "df_salary_tech_usd['timestamp'] = pd.to_datetime(df_salary_tech_usd['timestamp'], errors='coerce')\n", + "\n", + "\n", + "df_recent = df_salary_tech_usd[\n", + " df_salary_tech_usd['timestamp'].notna() &\n", + " df_salary_tech_usd['timestamp'].dt.year.isin([2021, 2022, 2023]) &\n", + " (df_salary_tech_usd['country'] == 'Us')\n", + "]\n", + "summary_recent = (\n", + " df_recent.groupby('state')['salary']\n", + " .agg(['min', 'mean', 'max'])\n", + " .sort_values('mean', ascending=False)\n", + ")\n", + "summary_recent\n" ] }, {