diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a6f0daa --- /dev/null +++ b/.gitignore @@ -0,0 +1,130 @@ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ \ No newline at end of file diff --git a/.yamllint.yaml b/.yamllint.yaml new file mode 100644 index 0000000..97975b5 --- /dev/null +++ b/.yamllint.yaml @@ -0,0 +1,46 @@ +# This config represents running the command `yamllint -d relaxed .` with the +# following extra rules: +# +# new-line-at-end-of-file: +# level: warning +# trailing-spaces: +# level: warning +# +# We also ignore the cookiecutter directories as these often contain +# jinja-style templating functions that yamllint doesn't play nicely with +# +# cribbed from https://github.com/2i2c-org/infrastructure/blob/main/.yamllint.yaml +--- +extends: default + +ignore: | + **/template/** + **/templates/** + +rules: + braces: + level: warning + max-spaces-inside: 1 + brackets: + level: warning + max-spaces-inside: 1 + colons: + level: warning + commas: + level: warning + comments: disable + comments-indentation: disable + document-start: disable + empty-lines: + level: warning + hyphens: + level: warning + indentation: + level: warning + indent-sequences: consistent + line-length: disable + new-line-at-end-of-file: + level: warning + trailing-spaces: + level: warning + truthy: disable \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 63286e3..289e9a0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # =================================================================== # Base image # =================================================================== -FROM buildpack-deps:jammy-scm AS base +FROM buildpack-deps:24.04 AS base # ------------------------------------------------------------------- # Environment diff --git a/README.md b/README.md index 5bce044..1cc1ac7 100644 --- a/README.md +++ b/README.md @@ -1 +1,35 @@ -# base-python-image \ No newline at end of file +# base-python-image + +This is the repo for DataHub Python base image and intended for DataHub admin use only. + +## Steps + +### Step 1: +- Admins make changes in apt.txt to manage system packages and environment.yaml to manage Conda and Pip packages. +- Once the PR is merged, the Github action for building the base image is triggered. + +### Step 2: +- In the Github logs, look for the new base image tag. Alternatively, you can navigate to the artifact registry on Google cloud console and find the tag for the latest image. +- Nagivate to a hub with the fancy profile configured (e.g. nature.datahub.berkeley.edu), and lauch a server with the newly built base image. +- Test the basic functionalities, including: + 1. Write a file in the terminal + 2. Open a notebook, write a notebook, run a notebook, and save a notebook. +- Test the recent changes, including: + 1. Versions (by running pip show \) + 2. New functionalities tied to a specific version + +### Step 3: +Once the base image is tested and everything looks good, manaully dispatch the "Trigger Downstream Builds" workflow in Github actions. + +1. Which downstream group to build? + + Choose "base" if you want to build the downstream R base image. + Choose "user" if you want to build the downstream hub specific user images. +2. Base image name (e.g. ucb-datahub-2018/testing/base-python-image) + + Use the base image name you have obtained in step 2. +3. Base image SHA/tag to propagate + + Use the base image tag you have obtained in step 2. + + diff --git a/apt.txt b/apt.txt index 0f49f0c..7164f98 100644 --- a/apt.txt +++ b/apt.txt @@ -7,12 +7,14 @@ rsync vim wget zip +tar screen man man-db manpages-posix manpages-dev manpages-posix-dev +tini # Regular build tools for compiling common stuff build-essential @@ -44,6 +46,6 @@ libxdamage1 libxfixes3 libxrandr2 libgbm1 -libasound2 +libasound2t64 diff --git a/environment.yml b/environment.yml index 9b626c5..dd70e9a 100644 --- a/environment.yml +++ b/environment.yml @@ -8,23 +8,33 @@ dependencies: # DH-418, bunch of stuff fails to install if git is set # higher to this e.g. git==2.47.1 - git + # vscode related + - jupyter-vscode-proxy==0.7 + - code-server==4.118.0 + # jupyter related - jupyter-archive==3.4.0 - jupyter-resource-usage==1.2.0 - jupyter-server-proxy==4.4.0 - jupyter_server==2.17.0 - jupyterhub==5.4.2 - - jupyterlab==4.5.3 + - jupyterlab==4.5.7 - jupyterlab-git==0.51.3 - jupyter-offlinenotebook==0.3.2 - jupyterlab_rise==0.43.1 - jupytext==1.18.1 - - nbgitpuller==1.2.2 - - notebook==7.5.3 - - pip==26.0.1 + - jupyterlab-favorites==3.5.0 + - jupyterlab_server==2.28.0 + - jupyterlab_widgets==3.0.16 + - syncthing==2.0.16 + - jupyter-syncthing-proxy==1.0.3 + - ipywidgets==8.1.8 + - nbgitpuller==1.3.0 + - notebook==7.5.6 - otter-grader==6.1.6 - wrapt==1.17.3 # Using python 3.11 for maximum compatibility with conda packages. - python==3.11 + - pip==26.0.1 # datascience packages - datascience==0.18.1 - numpy==2.4.2 @@ -35,11 +45,18 @@ dependencies: - seaborn==0.13.2 - statsmodels==0.14.6 - matplotlib==3.10.3 + - matplotlib-inline==0.2.1 + - ipympl==0.10.0 + - sympy==1.14.0 # DH-654 conflicts with git-crendential-helpers - pyjwt==2.10.1 + - nbclassic==1.3.3 + - sqlite==3.53.1 # For https://github.com/berkeley-dsep-infra/datahub/issues/1846 # Conda does not have these - pip: - git-credential-helpers==0.2 - nbconvert[webpdf]==7.17.0 + - nb2pdf==0.6.2 + - nbpdfexport==0.2.1 diff --git a/install-miniforge.bash b/install-miniforge.bash index 665f880..78f5d0f 100644 --- a/install-miniforge.bash +++ b/install-miniforge.bash @@ -3,7 +3,7 @@ set -ex cd $(dirname $0) -MINIFORGE_VERSION='26.1.0-0' +MINIFORGE_VERSION='26.3.2-0' URL="https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-x86_64.sh" INSTALLER_PATH=/tmp/miniforge-installer.sh