From e2a771fbbefa949f55f75ef0f6385cf3e79e531f Mon Sep 17 00:00:00 2001 From: havok2063 Date: Thu, 12 Jun 2025 11:43:16 -0400 Subject: [PATCH 01/10] adding docker dev files --- docker/Dockerfile.dev | 49 ++++++++++++++++++++++++++++++ docker/README.md | 11 ++++++- docker/docker-compose-dev.yml | 56 +++++++++++++++++++++++++++++++++++ 3 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 docker/Dockerfile.dev create mode 100644 docker/docker-compose-dev.yml diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev new file mode 100644 index 00000000..05e82020 --- /dev/null +++ b/docker/Dockerfile.dev @@ -0,0 +1,49 @@ +# Use an official Python runtime as a parent image +FROM python:3.10-slim + +# Set the working directory in the container +WORKDIR /app + +# Install library files +RUN apt-get update && \ + apt-get install -y --no-install-recommends git libpq-dev gcc build-essential mime-support && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Copy the repo contents into the container +COPY ./ /app + +# Install any needed packages +RUN pip install --no-cache-dir ".[web,db]" + +# Resolve some dependency issues +RUN pip install gunicorn +RUN pip install "itsdangerous==2.0.1" +RUN pip install "pillow<10" +RUN pip install "marshmallow<4.0.0" + +# Create the log directory and ensure it's writable +RUN mkdir -p /tmp/marvin/logs && \ + chmod -R 755 /tmp/marvin + +# Make port 8000 available to the world outside this container +EXPOSE 8000 + +# Copy the gunicorn config file into the container +COPY docker/gunicorn_config.py /app/python/gunicorn_config.py + +# Create the marvin config file +RUN mkdir -p /root/.marvin && chmod -R 755 /root/.marvin +RUN echo "use_sentry: False\nadd_github_message: False\ncheck_access: False" > /root/.marvin/marvin.yml + +# Update permissions +RUN chmod -R 755 /app/python/marvin/web + +# Set environment variables +ENV FLASK_APP="marvin.web.uwsgi_conf_files.app:app" + +# Change to python dir +WORKDIR /app/python + +# Run the application with uWSGI +CMD ["gunicorn", "-c", "gunicorn_config.py", "marvin.web.uwsgi_conf_files.app:app"] diff --git a/docker/README.md b/docker/README.md index ea7dcf8f..e0bc741c 100644 --- a/docker/README.md +++ b/docker/README.md @@ -50,4 +50,13 @@ The final web application is available at `localhost:8080/marvin` By default the marvin docker build will checkout the latest `main` branch of marvin on github and tag the image as `marvin:latest`. To change this to a specific branch or tag of marvin, e.g. `2.8.0` or `branch_name`, set the `MARVIN_TAG` environment variable. Then rebuild the system with `docker compose up --build`. This will checkout that branch or tag of the marvin repository and create -a new marvin docker image with that tag name. \ No newline at end of file +a new marvin docker image with that tag name. + +## Developer Notes + +To run the local developer docker, run the following command from the top level marvin project directory: +```bash +docker compose -f docker/docker-compose-dev.yml up --build +``` + +If you do not have a local postgres manga database, you can create a tunnel to the manga db at JHU. \ No newline at end of file diff --git a/docker/docker-compose-dev.yml b/docker/docker-compose-dev.yml new file mode 100644 index 00000000..74db069b --- /dev/null +++ b/docker/docker-compose-dev.yml @@ -0,0 +1,56 @@ +version: '3.9' +name: marvin +services: + nginx: + container_name: nginx + image: nginx:latest + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf + - socket_logs:/tmp/marvin + - web_assets:/usr/share/nginx/html + ports: + - 8080:80 + depends_on: + - marvin + networks: + - frontend + + redis: + container_name: redis + image: redis:latest + networks: + - backend + + marvin: + container_name: marvin + build: + context: .. + dockerfile: docker/Dockerfile.dev + args: + MARVIN_TAG: ${MARVIN_TAG:-main} + image: marvin:${MARVIN_TAG:-latest} + ports: + - 8000:8000 + volumes: + - ${SAS_BASE_DIR}:/root/sas/ + - $HOME/.pgpass:/root/.pgpass + - socket_logs:/tmp/marvin + - web_assets:/app/python/marvin/web + environment: + - SESSION_REDIS=redis://redis:6379 + - MARVIN_BASE=marvin + - PUBLIC_SERVER=True + - MANGADB_CONFIG=docker + networks: + - backend + - frontend + depends_on: + - redis + +networks: + backend: + frontend: + +volumes: + socket_logs: + web_assets: From 58d1dd9ac4faa5a3a45c7d227f1d5ddc652ad27c Mon Sep 17 00:00:00 2001 From: havok2063 Date: Thu, 12 Jun 2025 11:44:06 -0400 Subject: [PATCH 02/10] updating gunicorn threads --- docker/gunicorn_config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/gunicorn_config.py b/docker/gunicorn_config.py index 5979a895..f3d4139c 100644 --- a/docker/gunicorn_config.py +++ b/docker/gunicorn_config.py @@ -3,6 +3,8 @@ socket_dir = os.getenv("MARVIN_SOCKET_DIR", '/tmp/marvin') bind = [f"unix:{socket_dir}/marvin.sock", "0.0.0.0:8000"] workers = 1 +worker_class = "gthread" +threads = 8 daemon = False errorlog = os.path.join(os.getenv("MARVIN_LOGS_DIR", '/tmp/marvin/logs'), 'marvin_app_error.log') accesslog = os.path.join(os.getenv("MARVIN_LOGS_DIR", '/tmp/marvin/logs'), 'marvin_app_access.log') From bef381dcd21c74dd2cefb1b0fc0052fec4d25001 Mon Sep 17 00:00:00 2001 From: havok2063 Date: Thu, 12 Jun 2025 11:44:47 -0400 Subject: [PATCH 03/10] fixing docker build issues, removing header login ui --- docker/Dockerfile | 7 ++++-- python/marvin/web/templates/header.html | 29 ------------------------- python/marvin/web/web_utils.py | 4 ++-- setup.cfg | 1 + 4 files changed, 8 insertions(+), 33 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index dbc0b86f..e37f1652 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,10 +22,9 @@ RUN pip install --no-cache-dir ".[web,db]" # Resolve some dependency issues RUN pip install gunicorn -RUN pip install "jinja2<3.1" -RUN pip install "packaging<21" RUN pip install "itsdangerous==2.0.1" RUN pip install "pillow<10" +RUN pip install "marshmallow<4.0.0" # Create the log directory and ensure it's writable RUN mkdir -p /tmp/marvin/logs && \ @@ -47,6 +46,10 @@ RUN chmod -R 755 /app/python/marvin/web # Set environment variables ENV FLASK_APP="marvin.web.uwsgi_conf_files.app:app" +# Set a label +LABEL org.opencontainers.image.source=https://github.com/sdss/marvin +LABEL org.opencontainers.image.description="marvin production image" + # Change to python dir WORKDIR /app/python diff --git a/python/marvin/web/templates/header.html b/python/marvin/web/templates/header.html index 4d7ae4f4..189677e0 100644 --- a/python/marvin/web/templates/header.html +++ b/python/marvin/web/templates/header.html @@ -61,35 +61,6 @@ - {% elif not current_user.is_authenticated %} - - {# Login #} - - {% else %} - - {# User Preferences Post Login #} -
- -
{% endif %} diff --git a/python/marvin/web/web_utils.py b/python/marvin/web/web_utils.py index 80637aaa..97e5514a 100644 --- a/python/marvin/web/web_utils.py +++ b/python/marvin/web/web_utils.py @@ -43,14 +43,14 @@ def check_request_for_release(request): def configFeatures(app): ''' Configure Flask Feature Flags ''' - app.config['FEATURE_FLAGS']['public'] = True if config.access == 'public' else False + app.config['FEATURE_FLAGS']['public'] = True def check_access(): ''' Check the access mode in the session ''' # check if on public server - public_server = request.environ.get('PUBLIC_SERVER', None) == 'True' + public_server = request.environ.get('PUBLIC_SERVER', True) == 'True' public_flag = public_server or current_app.config['FEATURE_FLAGS']['public'] current_app.config['FEATURE_FLAGS']['public'] = public_server public_access = config.access == 'public' diff --git a/setup.cfg b/setup.cfg index 2509b41d..3516eb0c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -149,6 +149,7 @@ web= redis>=3.3 #uwsgi>=2.0.15 # install with "conda install -c conda-forge uwsgi" instead # added for rtd issues + marshmallow<4.0 validators>=0.10.3 intervals>=0.8.0 db= From 24601771d1d64d188356abe624aead6524b9a968 Mon Sep 17 00:00:00 2001 From: havok2063 Date: Thu, 12 Jun 2025 11:45:07 -0400 Subject: [PATCH 04/10] adding docker build github action --- .github/workflows/docker.yml | 54 ++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 .github/workflows/docker.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000..106fb46e --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,54 @@ +name: Docker + +on: + push: + tags: + - '[0-9]+.[0-9]+.[0-9]+' # Exclude pre-releases + branches: + - main + - dockertweaks + +jobs: + docker: + runs-on: ubuntu-latest + env: + USER: sdss + APP: marvin + steps: + - name: Set docker tags + id: set-tags + run: | + echo TAG_LATEST=$USER/$APP:latest >> $GITHUB_OUTPUT + if [[ $GITHUB_REF == refs/heads/main ]] + then + echo TAGS=$USER/$APP:latest >> $GITHUB_OUTPUT + elif [[ $GITHUB_REF == refs/heads/* ]] + then + BRANCH=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's/[\/]/_/g') + echo TAGS=$USER/$APP:$BRANCH >> $GITHUB_OUTPUT + else + echo TAGS=$USER/$APP:${GITHUB_REF#refs/tags/} >> $GITHUB_OUTPUT + fi + - name: Show tags + run: echo ${{ steps.set-tags.outputs.TAGS }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Log in to registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push + id: docker_build + uses: docker/build-push-action@v4 + with: + push: true + tags: | + ghcr.io/${{ steps.set-tags.outputs.TAGS }} + ghcr.io/${{ steps.set-tags.outputs.TAG_LATEST }} + file: ./Dockerfile + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} \ No newline at end of file From bf5cbc5190bca89895dda7dd650dd8f61332b384 Mon Sep 17 00:00:00 2001 From: havok2063 Date: Thu, 12 Jun 2025 11:48:46 -0400 Subject: [PATCH 05/10] tweak to action --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 106fb46e..47e3d622 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -49,6 +49,6 @@ jobs: tags: | ghcr.io/${{ steps.set-tags.outputs.TAGS }} ghcr.io/${{ steps.set-tags.outputs.TAG_LATEST }} - file: ./Dockerfile + file: ./docker/Dockerfile - name: Image digest run: echo ${{ steps.docker_build.outputs.digest }} \ No newline at end of file From 6e77db811b9c56683a2b198be6f427d8666de228 Mon Sep 17 00:00:00 2001 From: havok2063 Date: Thu, 12 Jun 2025 11:54:10 -0400 Subject: [PATCH 06/10] test action context --- .github/workflows/docker.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 47e3d622..2df4c632 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -15,6 +15,8 @@ jobs: USER: sdss APP: marvin steps: + - name: Checkout + uses: actions/checkout@v4 - name: Set docker tags id: set-tags run: | @@ -49,6 +51,7 @@ jobs: tags: | ghcr.io/${{ steps.set-tags.outputs.TAGS }} ghcr.io/${{ steps.set-tags.outputs.TAG_LATEST }} - file: ./docker/Dockerfile + context: docker + file: Dockerfile - name: Image digest run: echo ${{ steps.docker_build.outputs.digest }} \ No newline at end of file From 791468bf67513b2a68c6e5d0dccf263208b62830 Mon Sep 17 00:00:00 2001 From: havok2063 Date: Thu, 12 Jun 2025 11:56:54 -0400 Subject: [PATCH 07/10] test action context --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2df4c632..b4e54b07 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -52,6 +52,6 @@ jobs: ghcr.io/${{ steps.set-tags.outputs.TAGS }} ghcr.io/${{ steps.set-tags.outputs.TAG_LATEST }} context: docker - file: Dockerfile + file: .docker/Dockerfile - name: Image digest run: echo ${{ steps.docker_build.outputs.digest }} \ No newline at end of file From 7aaaffe442b4ef4506ff2fd0e2835ce137c74a70 Mon Sep 17 00:00:00 2001 From: havok2063 Date: Thu, 12 Jun 2025 11:57:40 -0400 Subject: [PATCH 08/10] test action context --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b4e54b07..a9b0a10f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -52,6 +52,6 @@ jobs: ghcr.io/${{ steps.set-tags.outputs.TAGS }} ghcr.io/${{ steps.set-tags.outputs.TAG_LATEST }} context: docker - file: .docker/Dockerfile + file: docker/Dockerfile - name: Image digest run: echo ${{ steps.docker_build.outputs.digest }} \ No newline at end of file From c98284cc9ad1b4a2cea2ad75aa9a047f55e9394f Mon Sep 17 00:00:00 2001 From: havok2063 Date: Thu, 12 Jun 2025 12:07:11 -0400 Subject: [PATCH 09/10] updating changelog --- .github/workflows/docker.yml | 1 - CHANGELOG.rst | 7 +++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a9b0a10f..157aca16 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -6,7 +6,6 @@ on: - '[0-9]+.[0-9]+.[0-9]+' # Exclude pre-releases branches: - main - - dockertweaks jobs: docker: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c87c2789..38cd3503 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,13 @@ Marvin's Change Log =================== + +[2.8.3] - 2025/06/12 +-------------------- +- Removed header UI user login to fix bug in web deploy +- Updated docker build to fix deps issues +- Adding docker github action + [2.8.2] - 2025/02/05 -------------------- - Updating build to comply with PEP 625 From 0347f75941e61fda6f1145baf3483f087926cfc5 Mon Sep 17 00:00:00 2001 From: havok2063 Date: Thu, 12 Jun 2025 12:07:50 -0400 Subject: [PATCH 10/10] =?UTF-8?q?Bump=20version:=202.8.2=20=E2=86=92=202.8?= =?UTF-8?q?.3dev?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 7 ++++--- setup.cfg | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 34d53350..55c861f2 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,17 +1,18 @@ [bumpversion] -current_version = 2.8.2 +current_version = 2.8.3dev commit = True tag = False tag_name = {new_version} parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z]+)? -serialize = +serialize = {major}.{minor}.{patch}{release} {major}.{minor}.{patch} [bumpversion:part:release] optional_value = alpha -values = +values = dev alpha [bumpversion:file:setup.cfg] + diff --git a/setup.cfg b/setup.cfg index 3516eb0c..c1676dbb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = sdss-marvin -version = 2.8.2 +version = 2.8.3dev author = The Marvin Developers author_email = havok2063@hotmail.com description = Toolsuite for dealing with the MaNGA dataset