From 200843042c0bec1a63e55d21423281eea5e982a3 Mon Sep 17 00:00:00 2001 From: Jon Massey Date: Tue, 29 Jul 2025 13:47:47 +0000 Subject: [PATCH 1/3] WIP --- .devcontainer/devcontainer.json | 17 ++++------------- .devcontainer/postAttach.sh | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 13 deletions(-) create mode 100644 .devcontainer/postAttach.sh diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 6e52b56..bc790fc 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,21 +2,12 @@ // README at: https://github.com/devcontainers/templates/tree/main/src/python { "name": "OpenSAFELY", - "image": "ghcr.io/opensafely-core/research-template:v0", + "image": "mcr.microsoft.com/devcontainers/python:3.10", // Features to add to the dev container. More info: https://containers.dev/features. "features": { "ghcr.io/devcontainers/features/docker-in-docker:2": {} }, - "postCreateCommand": "/bin/bash /opt/devcontainer/postCreate.sh ${containerWorkspaceFolder}", - "postAttachCommand": "/bin/bash /opt/devcontainer/postAttach.sh", - "forwardPorts": [ - 8787 - ], - "portsAttributes": { - "8787": { - "label": "RStudio IDE" - } - }, + "postAttachCommand": "/bin/bash .devcontainer/postAttach.sh", // Configure tool-specific properties. "customizations": { "codespaces": { @@ -40,8 +31,8 @@ "files.autoSave": "afterDelay", "files.autoSaveDelay": 1000, "git.autofetch": true, - "python.analysis.extraPaths": [".devcontainer/ehrql-main/"], - "python.defaultInterpreterPath": "/opt/venv/bin/python", + "python.analysis.extraPaths": [".ehrql/"], + // "python.defaultInterpreterPath": "/opt/venv/bin/python", "python.terminal.activateEnvInCurrentTerminal": true, "python.terminal.activateEnvironment": true, "window.autoDetectColorScheme": true diff --git a/.devcontainer/postAttach.sh b/.devcontainer/postAttach.sh new file mode 100644 index 0000000..d44e23a --- /dev/null +++ b/.devcontainer/postAttach.sh @@ -0,0 +1,17 @@ +pip install opensafely +docker run -it -d --name ehrql --rm ghcr.io/opensafely-core/ehrql:v1 bash +mount=$(docker inspect ehrql -f '{{.GraphDriver.Data.MergedDir}}') +mkdir -p .ehrql +sudo mount --bind -o ro "$mount/app/ehrql" .ehrql +sudo touch /usr/bin/R +sudo mkdir -p /usr/lib/R +sudo mkdir -p /etc/R +docker run -it -d --name r-v2 --rm ghcr.io/opensafely-core/r:v2 bash +mount=$(docker inspect r-v2 -f '{{.GraphDriver.Data.MergedDir}}') +sudo mount --bind -o ro "$mount/usr/bin/R" /usr/bin/R +sudo mount --bind -o ro "$mount/usr/lib/R" /usr/lib/R +sudo mount --bind -o ro "$mount/etc/R" /etc/R +docker run -it -d --name python-v2 --rm ghcr.io/opensafely-core/python:v2 bash +mount=$(docker inspect python-v2 -f '{{.GraphDriver.Data.MergedDir}}') +sudo mkdir -p /opt/venv +sudo mount --bind -o ro "$mount/opt/venv" /opt/venv \ No newline at end of file From af8383d6588486d5b4e0d4ec0e50d607735bea62 Mon Sep 17 00:00:00 2001 From: Jon Massey Date: Mon, 4 Aug 2025 18:43:00 +0000 Subject: [PATCH 2/3] Use bindfs for mounts --- .devcontainer/devcontainer.json | 3 ++- .devcontainer/postAttach.sh | 12 ++++++++---- .gitignore | 1 + 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index bc790fc..ec5386f 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -8,6 +8,7 @@ "ghcr.io/devcontainers/features/docker-in-docker:2": {} }, "postAttachCommand": "/bin/bash .devcontainer/postAttach.sh", + "containerEnv": {"R_LD_LIBRARY_PATH":"/usr/lib/R-system"}, // Configure tool-specific properties. "customizations": { "codespaces": { @@ -32,7 +33,7 @@ "files.autoSaveDelay": 1000, "git.autofetch": true, "python.analysis.extraPaths": [".ehrql/"], - // "python.defaultInterpreterPath": "/opt/venv/bin/python", + "python.defaultInterpreterPath": "/opt/venv/bin/python3.10", "python.terminal.activateEnvInCurrentTerminal": true, "python.terminal.activateEnvironment": true, "window.autoDetectColorScheme": true diff --git a/.devcontainer/postAttach.sh b/.devcontainer/postAttach.sh index d44e23a..3615106 100644 --- a/.devcontainer/postAttach.sh +++ b/.devcontainer/postAttach.sh @@ -1,17 +1,21 @@ pip install opensafely +sudo apt update +sudo apt install -y bindfs docker run -it -d --name ehrql --rm ghcr.io/opensafely-core/ehrql:v1 bash mount=$(docker inspect ehrql -f '{{.GraphDriver.Data.MergedDir}}') mkdir -p .ehrql -sudo mount --bind -o ro "$mount/app/ehrql" .ehrql +sudo mount -o bind "$mount/app/ehrql" .ehrql sudo touch /usr/bin/R sudo mkdir -p /usr/lib/R +sudo mkdir -p /usr/lib/R-system sudo mkdir -p /etc/R docker run -it -d --name r-v2 --rm ghcr.io/opensafely-core/r:v2 bash mount=$(docker inspect r-v2 -f '{{.GraphDriver.Data.MergedDir}}') sudo mount --bind -o ro "$mount/usr/bin/R" /usr/bin/R -sudo mount --bind -o ro "$mount/usr/lib/R" /usr/lib/R -sudo mount --bind -o ro "$mount/etc/R" /etc/R +sudo bindfs -r --resolve-symlinks "$mount/usr/lib/R" /usr/lib/R +sudo bindfs -r --resolve-symlinks "$mount/etc/R" /etc/R +sudo bindfs -r --resolve-symlinks "$mount/lib" /usr/lib/R-system docker run -it -d --name python-v2 --rm ghcr.io/opensafely-core/python:v2 bash mount=$(docker inspect python-v2 -f '{{.GraphDriver.Data.MergedDir}}') sudo mkdir -p /opt/venv -sudo mount --bind -o ro "$mount/opt/venv" /opt/venv \ No newline at end of file +sudo bindfs -r --resolve-symlinks "$mount/opt/venv" /opt/venv \ No newline at end of file diff --git a/.gitignore b/.gitignore index 1c206cf..8577286 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ venv/ .Rhistory .Rproj.user/ .devcontainer/ehrql-main/ +.ehrql \ No newline at end of file From fa559a3e1e07f02fe531dcc553d1d1a1fc3a5e5c Mon Sep 17 00:00:00 2001 From: Jon Massey Date: Wed, 6 Aug 2025 22:30:49 +0000 Subject: [PATCH 3/3] Working version using docker cp rather than mounts Base on MS devcontainer Jammy image to match OpenSAFELY base image. Install R/Python and their dependencies. Use `docker cp` to copy the R/Python library files to the host filesystem which avoids any of the previously seen issues with mounting a running container. These library directories don't contain any symlinks (verified with `find . -type l`). --- .devcontainer/devcontainer.json | 14 +++++++++++--- .devcontainer/postAttach.sh | 22 +--------------------- .devcontainer/postCreate.sh | 26 ++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 24 deletions(-) create mode 100644 .devcontainer/postCreate.sh diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index ec5386f..9a56901 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,13 +2,21 @@ // README at: https://github.com/devcontainers/templates/tree/main/src/python { "name": "OpenSAFELY", - "image": "mcr.microsoft.com/devcontainers/python:3.10", + "image": "mcr.microsoft.com/devcontainers/base:ubuntu-22.04", // Features to add to the dev container. More info: https://containers.dev/features. "features": { "ghcr.io/devcontainers/features/docker-in-docker:2": {} }, + "postCreateCommand": "/bin/bash .devcontainer/postCreate.sh", "postAttachCommand": "/bin/bash .devcontainer/postAttach.sh", - "containerEnv": {"R_LD_LIBRARY_PATH":"/usr/lib/R-system"}, + "forwardPorts": [ + 8787 + ], + "portsAttributes": { + "8787": { + "label": "RStudio IDE" + } + }, // Configure tool-specific properties. "customizations": { "codespaces": { @@ -33,7 +41,7 @@ "files.autoSaveDelay": 1000, "git.autofetch": true, "python.analysis.extraPaths": [".ehrql/"], - "python.defaultInterpreterPath": "/opt/venv/bin/python3.10", + "python.defaultInterpreterPath": "./venv/bin/python", "python.terminal.activateEnvInCurrentTerminal": true, "python.terminal.activateEnvironment": true, "window.autoDetectColorScheme": true diff --git a/.devcontainer/postAttach.sh b/.devcontainer/postAttach.sh index 3615106..36ef0ff 100644 --- a/.devcontainer/postAttach.sh +++ b/.devcontainer/postAttach.sh @@ -1,21 +1 @@ -pip install opensafely -sudo apt update -sudo apt install -y bindfs -docker run -it -d --name ehrql --rm ghcr.io/opensafely-core/ehrql:v1 bash -mount=$(docker inspect ehrql -f '{{.GraphDriver.Data.MergedDir}}') -mkdir -p .ehrql -sudo mount -o bind "$mount/app/ehrql" .ehrql -sudo touch /usr/bin/R -sudo mkdir -p /usr/lib/R -sudo mkdir -p /usr/lib/R-system -sudo mkdir -p /etc/R -docker run -it -d --name r-v2 --rm ghcr.io/opensafely-core/r:v2 bash -mount=$(docker inspect r-v2 -f '{{.GraphDriver.Data.MergedDir}}') -sudo mount --bind -o ro "$mount/usr/bin/R" /usr/bin/R -sudo bindfs -r --resolve-symlinks "$mount/usr/lib/R" /usr/lib/R -sudo bindfs -r --resolve-symlinks "$mount/etc/R" /etc/R -sudo bindfs -r --resolve-symlinks "$mount/lib" /usr/lib/R-system -docker run -it -d --name python-v2 --rm ghcr.io/opensafely-core/python:v2 bash -mount=$(docker inspect python-v2 -f '{{.GraphDriver.Data.MergedDir}}') -sudo mkdir -p /opt/venv -sudo bindfs -r --resolve-symlinks "$mount/opt/venv" /opt/venv \ No newline at end of file +opensafely launch rstudio -p 8787 & \ No newline at end of file diff --git a/.devcontainer/postCreate.sh b/.devcontainer/postCreate.sh new file mode 100644 index 0000000..0e9ee0f --- /dev/null +++ b/.devcontainer/postCreate.sh @@ -0,0 +1,26 @@ +# install python and dependencies, set up virtualenv +sudo apt update +wget https://github.com/opensafely-core/python-docker/raw/refs/heads/main/v2/dependencies.txt -q -O - | sed 's/^#.*//' | sudo xargs apt-get install --yes --no-install-recommends +pip install virtualenv opensafely +python3 -m virtualenv .venv +rm -rf .venv/lib +docker container create --name python-v2 ghcr.io/opensafely-core/python:v2 +docker cp python-v2:/opt/venv/lib .venv/ +docker rm python-v2 + +# install R and dependencies +echo "deb https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/" | sudo tee /etc/apt/sources.list.d/cran.list >/dev/null +sudo /usr/lib/apt/apt-helper download-file 'https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc' /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc +sudo apt update +wget https://raw.githubusercontent.com/opensafely-core/r-docker/refs/heads/main/v2/dependencies.txt -q -O - | sed 's/^#.*//' | sudo xargs apt-get install --yes --no-install-recommends +wget https://raw.githubusercontent.com/opensafely-core/r-docker/refs/heads/main/scripts/rprofile-site-append-1.R -q -O - | sudo tee -a /usr/lib/R/etc/Rprofile.site >/dev/null +wget https://raw.githubusercontent.com/opensafely-core/r-docker/refs/heads/main/scripts/rprofile-site-append-2.R -q -O - | sudo tee -a /usr/lib/R/etc/Rprofile.site >/dev/null +docker create --name r-v2 --rm ghcr.io/opensafely-core/r:v2 +sudo docker cp r-v2:/usr/local/lib/R/site-library /usr/local/lib/R/ +sudo chown -R vscode:vscode /usr/local/lib/R/site-library/ + + +#download and extract latest ehrql source +wget https://github.com/opensafely-core/ehrql/archive/main.zip -P .devcontainer +unzip -o .devcontainer/main.zip -d .devcontainer/ +rm .devcontainer/main.zip \ No newline at end of file