From 622b8fcbe5c4ee1ab3113bd1b6c79e704a2212e8 Mon Sep 17 00:00:00 2001 From: Naexon Date: Fri, 10 Mar 2023 11:36:37 +0100 Subject: [PATCH 01/28] added cli submodule --- .gitmodules | 3 +++ execDAT-CLI | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 execDAT-CLI diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..66c65c0 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "execDAT-CLI"] + path = execDAT-CLI + url = git@github.com:AustrianDataLAB/execDAT-CLI.git diff --git a/execDAT-CLI b/execDAT-CLI new file mode 160000 index 0000000..7b61083 --- /dev/null +++ b/execDAT-CLI @@ -0,0 +1 @@ +Subproject commit 7b6108365887b7cde415b7580fd5d806a38e3f12 From eef8c28762123231a904886ef32ecd59e08f959c Mon Sep 17 00:00:00 2001 From: Thomas Weber Date: Thu, 16 Mar 2023 13:54:29 +0100 Subject: [PATCH 02/28] use dev branch of execDAT-CLI --- .gitmodules | 1 + execDAT-CLI | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 66c65c0..4664fce 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,4 @@ [submodule "execDAT-CLI"] path = execDAT-CLI url = git@github.com:AustrianDataLAB/execDAT-CLI.git + branch = dev diff --git a/execDAT-CLI b/execDAT-CLI index 7b61083..34b74a6 160000 --- a/execDAT-CLI +++ b/execDAT-CLI @@ -1 +1 @@ -Subproject commit 7b6108365887b7cde415b7580fd5d806a38e3f12 +Subproject commit 34b74a66fa7cede7367f2104bc9628d245cfcd84 From 3b76edd1fae32ee5b0ed27aa7d460beac6c38116 Mon Sep 17 00:00:00 2001 From: Thomas Weber Date: Thu, 16 Mar 2023 14:01:59 +0100 Subject: [PATCH 03/28] use main branch for checkout of cli in main --- .gitmodules | 2 +- docs/adrs/.gitignore | 0 docs/images/.gitignore | 0 3 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 docs/adrs/.gitignore create mode 100644 docs/images/.gitignore diff --git a/.gitmodules b/.gitmodules index 4664fce..842b27a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "execDAT-CLI"] path = execDAT-CLI url = git@github.com:AustrianDataLAB/execDAT-CLI.git - branch = dev + branch = main diff --git a/docs/adrs/.gitignore b/docs/adrs/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/docs/images/.gitignore b/docs/images/.gitignore new file mode 100644 index 0000000..e69de29 From c1e418f99c19096a5bd5ef80bd4aa504c3e75bb5 Mon Sep 17 00:00:00 2001 From: Naexon Date: Thu, 16 Mar 2023 14:39:21 +0100 Subject: [PATCH 04/28] Template for the ADRs We adapted an ADR template we found online. Some changes are unique to our template: We are using the date as prefix in the ADR folder for natural ordering. Also, we require a link to the corresponding ADR in deprecated and superseded ADRs. --- docs/adrs/YYYY-MM-DD-template.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 docs/adrs/YYYY-MM-DD-template.md diff --git a/docs/adrs/YYYY-MM-DD-template.md b/docs/adrs/YYYY-MM-DD-template.md new file mode 100644 index 0000000..2c75292 --- /dev/null +++ b/docs/adrs/YYYY-MM-DD-template.md @@ -0,0 +1,26 @@ +# Title + +Date: YYYY-MM-DD + +## Status + +What is the status if the ADR? + +Possible options: +* __PROPOSED__ +* __ACCEPTED__ +* __REJECTED__ +* __DEPRECATED__ (include reference to the superseding ADR) +* __SUPERSEDED__ (include reference to the deprecating ADR) + +## Context + +What is the context of this ADR? What is the issue that we are seeing? What is motivating this decision or change? + +## Decision + +What is the change we are proposing? What do we plan on doing to solve the issue? + +## Consequences + +What are the consequences of the change? What will be more difficult? What will be easier? \ No newline at end of file From 2dbb93e35f61171c9c512796369fa583a54bcac7 Mon Sep 17 00:00:00 2001 From: Thomas Weber Date: Thu, 16 Mar 2023 14:26:06 +0100 Subject: [PATCH 05/28] add index and valueproposition --- docs/Index.md | 5 +++ docs/ValueProposition.md | 91 ++++++++++++++++++++++++++++++++++++++++ execDAT-CLI | 2 +- 3 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 docs/Index.md create mode 100644 docs/ValueProposition.md diff --git a/docs/Index.md b/docs/Index.md new file mode 100644 index 0000000..f884d33 --- /dev/null +++ b/docs/Index.md @@ -0,0 +1,5 @@ +# ExecDat + +Table of Contents + +- [Value Proposition](./ValueProposition.md) diff --git a/docs/ValueProposition.md b/docs/ValueProposition.md new file mode 100644 index 0000000..09dc035 --- /dev/null +++ b/docs/ValueProposition.md @@ -0,0 +1,91 @@ +# ExecDat - Value Proposition + +## What is the core value being generated? + +The goal of this project is to provide researchers with an easy and efficient way to execute and verify scientific evaluations, regardless of whether the required data and code is local or remote. + +This will be achieved through the development of a user-friendly tool, which could take the form of a CLI tool, kubectl plugin, or API endpoint. The tool will enable easy remote execution of code that requires access to remote or local datasets, simplifying the research process and reducing the technical barriers to entry. + +## Team + +### Project owner / Deputy owner + +DAT Team + +### Team members + +Daniel Hofstätter, Alexander Woda and Thomas Weber + +## Problem Space + +Why are we doing this? How do we judge success? + +### Problem statement + +Researchers face difficulties executing code that requires access to remote or local datasets. + +I.e., executing scientific evaluations on those datasets exclusively local might be problematic because users face large dataset sizes and have certain dependencies on, for example Operating Systems or Hardware. Furthermore, the current coupling of code to local hardware leads to limitations in parallel executions, resulting in high evaluation and iteration times. + +### Impact of this problem + +The impact of the problem is that it can slow down the progress of research and create barriers to entry for researchers with limited technical expertise. The manual setup and management of research environments can be time-consuming, distracting, and prone to errors. This can limit the ability of researchers to explore and analyze data, and ultimately, hinder the development of new scientific insights and breakthroughs. The impact is especially significant in fields such as data science and machine learning, where access to large and complex datasets is crucial for research. + +E.g., imagine a scientific paper is published, or going to be published, and reviewers want to verify results in them, maybe even for different datasets. Downloading Gigabytes of data or demanding hours of runtime on limited hardware slows down the review process. + +### Who is the customer/ target audience + +The target audiences for the proposed software tool are researchers and scientists who require access to remote or local datasets for their research. This includes researchers in fields such as data science, machine learning, and other areas that require extensive data analysis. + +For Example: + +Everyone interested in research, but with an initial scope limited to Universities (Professors, students, etc.) +Universities to host our service and provide access to staff +Research teams at any organization + +### Criteria for Success + +We provide simplicity of execution, reusability of environments, proofable validity of results and asynchronicity in the evaluation process. Our solution is to create a user-friendly software tool that simplifies the process, reduces time and effort, and allows researchers to focus on their research questions. + +According to these goals, we define the following criteria: + +Usability: One simple function call should be enough. +Scalability: Multiple users should be able to do evaluations in parallel. +Flexibility: Should support multiple languages and a variety of operating systems. +Repeatability: Different users should get the same results for the same evaluation. + +## MVP + +### What needs to be true in order for a prototype to be ready for release? + +We can ship an MVP to researchers and scientists, as soon as we have the following MUST-HAVEs: + +#### Functional MUST-HAVEs + +Remote code execution: The tool enables remote execution of code that requires access to remote datasets. + +Parallel execution support: The tool supports parallel execution of scientific evaluations to reduce evaluation and iteration times. + +Result size: A maximum size of one Gigabyte in the result file is supported. +User interaction: Only one CLI command and one config file is needed to run a job. +E.g., "execDAT " or "kubectl apply -f " + +#### Non-Functional MUST-HAVEs + +Flexibility: We support at least two different environment configurations. +Scalability: Scales to at least two users each having at least two jobs running. +Validity of results: Two users executing with the same configuration file get the same result. + +### What crucial factors are we missing? + +Definition of work packages +Technical Overview Diagram +Cluster Access + +### What is the key question we would ask to understand if we are on the right track? + +Do we simplify the research process? +In a side-by-side comparison, are users preferring to use our service, compared to a local execution of the task on their hardware? + +### Who are the alpha testers that we can use for validating our assumptions? + +DAT Team diff --git a/execDAT-CLI b/execDAT-CLI index 34b74a6..84dd8f5 160000 --- a/execDAT-CLI +++ b/execDAT-CLI @@ -1 +1 @@ -Subproject commit 34b74a66fa7cede7367f2104bc9628d245cfcd84 +Subproject commit 84dd8f5b66d070aefa88eab195a7aa7fb0cf24cb From 69d9a2cfe100a8292df467ff2069d422e9787e8a Mon Sep 17 00:00:00 2001 From: Thomas Weber Date: Thu, 16 Mar 2023 15:25:04 +0100 Subject: [PATCH 06/28] add operator as submodule --- .gitmodules | 6 +++++- execDAT-operator | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) create mode 160000 execDAT-operator diff --git a/.gitmodules b/.gitmodules index 842b27a..1bfe593 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,8 @@ [submodule "execDAT-CLI"] path = execDAT-CLI url = git@github.com:AustrianDataLAB/execDAT-CLI.git - branch = main + branch = main +[submodule "execDAT-operator"] + path = execDAT-operator + url = git@github.com:AustrianDataLAB/execDAT-operator.git + branch = main diff --git a/execDAT-operator b/execDAT-operator new file mode 160000 index 0000000..876b051 --- /dev/null +++ b/execDAT-operator @@ -0,0 +1 @@ +Subproject commit 876b0517e8fa7f137dcb98a1fcf1781b8e1a1946 From 1234aacffd85d90d43d416790c016afbd34b2fee Mon Sep 17 00:00:00 2001 From: Thomas Weber Date: Thu, 16 Mar 2023 16:10:48 +0100 Subject: [PATCH 07/28] add architecture and local k3d setup --- README.md | 14 ++ docs/drawio/architecture_overview.drawio | 294 +++++++++++++++++++++++ k3d-dev.yaml | 47 ++++ 3 files changed, 355 insertions(+) create mode 100644 docs/drawio/architecture_overview.drawio create mode 100644 k3d-dev.yaml diff --git a/README.md b/README.md index 622d3d9..2eab463 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,16 @@ # execDAT + execDAT - remote code execution for research + +## Getting Started + +### Prerequisites + +* k3d +* docker + +### Start k3d cluster + +```shell +k3d cluster create -c k3d-dev.yaml +``` diff --git a/docs/drawio/architecture_overview.drawio b/docs/drawio/architecture_overview.drawio new file mode 100644 index 0000000..2b36fa8 --- /dev/null +++ b/docs/drawio/architecture_overview.drawio @@ -0,0 +1,294 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/k3d-dev.yaml b/k3d-dev.yaml new file mode 100644 index 0000000..9b55113 --- /dev/null +++ b/k3d-dev.yaml @@ -0,0 +1,47 @@ +--- +apiVersion: k3d.io/v1alpha4 +kind: Simple +metadata: + name: execdat-dev +servers: 1 +agents: 0 +image: docker.io/rancher/k3s:v1.25.6-k3s1 + +# kubeAPI: # same as `--api-port myhost.my.domain:6445` (where the name would resolve to 127.0.0.1) +# host: "myhost.my.domain" # important for the `server` setting in the kubeconfig +# hostIP: "127.0.0.1" # where the Kubernetes API will be listening on +# hostPort: "6445" # where the Kubernetes API listening port will be mapped to on your host system +network: k3d-network # same as `--network my-custom-net` +ports: + - port: 8080:80 # same as `--port '8080:80@loadbalancer'` + nodeFilters: + - loadbalancer + +# registries: # define how registries should be created or used +# create: # creates a default registry to be used with the cluster; same as `--registry-create registry.localhost` +# name: registry.localhost +# host: "0.0.0.0" +# hostPort: "5000" +# proxy: # omit this to have a "normal" registry, set this to create a registry proxy (pull-through cache) +# remoteURL: https://registry-1.docker.io # mirror the DockerHub registry +# username: "" # unauthenticated +# password: "" # unauthenticated +# volumes: +# - /some/path:/var/lib/registry # persist registry data locally +# use: +# - k3d-myotherregistry:5000 # some other k3d-managed registry; same as `--registry-use 'k3d-myotherregistry:5000'` +# config: | # define contents of the `registries.yaml` file (or reference a file); same as `--registry-config /path/to/config.yaml` +# mirrors: +# "my.company.registry": +# endpoint: +# - http://my.company.registry:5000 + +options: + k3s: # options passed on to K3s itself + extraArgs: # additional arguments passed to the `k3s server|agent` command; same as `--k3s-arg` + - arg: "--tls-san=k3d.localhost" + nodeFilters: + - server:* + kubeconfig: + updateDefaultKubeconfig: true # add new cluster to your default Kubeconfig; same as `--kubeconfig-update-default` (default: true) + switchCurrentContext: true # also set current-context to the new cluster's context; same as `--kubeconfig-switch-context` (default: true) From 83212286d3a5d2584f8784c6b482603ea834b7ac Mon Sep 17 00:00:00 2001 From: Naexon <75274749+Naexon@users.noreply.github.com> Date: Thu, 16 Mar 2023 16:14:56 +0100 Subject: [PATCH 08/28] Template for the ADRs (#6) We adapted an ADR template we found online. Some changes are unique to our template: We are using the date as prefix in the ADR folder for natural ordering. Also, we require a link to the corresponding ADR in deprecated and superseded ADRs. --- execDAT-operator | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/execDAT-operator b/execDAT-operator index 876b051..5b80818 160000 --- a/execDAT-operator +++ b/execDAT-operator @@ -1 +1 @@ -Subproject commit 876b0517e8fa7f137dcb98a1fcf1781b8e1a1946 +Subproject commit 5b8081847bbfcc88e90dc41c4417f6fe32ae52c7 From 9a47217cd64ba377f4aff175ddad315150d938cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Hofst=C3=A4tter?= Date: Thu, 16 Mar 2023 17:10:44 +0100 Subject: [PATCH 09/28] public vs private ADR Co-authored-by: Thomas Weber --- .../adrs/2023-03-16-public-vs-private-data.md | 18 +++++++++++++ docs/adrs/2023-03-DD-cicd-solution.md | 26 +++++++++++++++++++ docs/adrs/2023-03-DD-regsitry-solution.md | 26 +++++++++++++++++++ .../2023-03-DD-storage-bucket-solution.md | 26 +++++++++++++++++++ 4 files changed, 96 insertions(+) create mode 100644 docs/adrs/2023-03-16-public-vs-private-data.md create mode 100644 docs/adrs/2023-03-DD-cicd-solution.md create mode 100644 docs/adrs/2023-03-DD-regsitry-solution.md create mode 100644 docs/adrs/2023-03-DD-storage-bucket-solution.md diff --git a/docs/adrs/2023-03-16-public-vs-private-data.md b/docs/adrs/2023-03-16-public-vs-private-data.md new file mode 100644 index 0000000..c58aed6 --- /dev/null +++ b/docs/adrs/2023-03-16-public-vs-private-data.md @@ -0,0 +1,18 @@ +# Decide if source code an data of users can be private. + +Date: 2023-03-16 + +## Status +__PROPOSED__ + +## Context + +The jobs need to access the data and source code of the user in order to create the image and run the task. Private repositories need additional user authentication whereas public ones don't. + +## Decision + +For now we only allow public code repositories and data sources. This means that the code and data of the user is public. This is the easiest way to implement the jobs. We can always change this later. + +## Consequences + +This means that the user has to make the code and data public. This is not a problem for the user, because the user wants to publish the code and data anyway. The user can always make the code and data private later. \ No newline at end of file diff --git a/docs/adrs/2023-03-DD-cicd-solution.md b/docs/adrs/2023-03-DD-cicd-solution.md new file mode 100644 index 0000000..2c75292 --- /dev/null +++ b/docs/adrs/2023-03-DD-cicd-solution.md @@ -0,0 +1,26 @@ +# Title + +Date: YYYY-MM-DD + +## Status + +What is the status if the ADR? + +Possible options: +* __PROPOSED__ +* __ACCEPTED__ +* __REJECTED__ +* __DEPRECATED__ (include reference to the superseding ADR) +* __SUPERSEDED__ (include reference to the deprecating ADR) + +## Context + +What is the context of this ADR? What is the issue that we are seeing? What is motivating this decision or change? + +## Decision + +What is the change we are proposing? What do we plan on doing to solve the issue? + +## Consequences + +What are the consequences of the change? What will be more difficult? What will be easier? \ No newline at end of file diff --git a/docs/adrs/2023-03-DD-regsitry-solution.md b/docs/adrs/2023-03-DD-regsitry-solution.md new file mode 100644 index 0000000..2c75292 --- /dev/null +++ b/docs/adrs/2023-03-DD-regsitry-solution.md @@ -0,0 +1,26 @@ +# Title + +Date: YYYY-MM-DD + +## Status + +What is the status if the ADR? + +Possible options: +* __PROPOSED__ +* __ACCEPTED__ +* __REJECTED__ +* __DEPRECATED__ (include reference to the superseding ADR) +* __SUPERSEDED__ (include reference to the deprecating ADR) + +## Context + +What is the context of this ADR? What is the issue that we are seeing? What is motivating this decision or change? + +## Decision + +What is the change we are proposing? What do we plan on doing to solve the issue? + +## Consequences + +What are the consequences of the change? What will be more difficult? What will be easier? \ No newline at end of file diff --git a/docs/adrs/2023-03-DD-storage-bucket-solution.md b/docs/adrs/2023-03-DD-storage-bucket-solution.md new file mode 100644 index 0000000..2c75292 --- /dev/null +++ b/docs/adrs/2023-03-DD-storage-bucket-solution.md @@ -0,0 +1,26 @@ +# Title + +Date: YYYY-MM-DD + +## Status + +What is the status if the ADR? + +Possible options: +* __PROPOSED__ +* __ACCEPTED__ +* __REJECTED__ +* __DEPRECATED__ (include reference to the superseding ADR) +* __SUPERSEDED__ (include reference to the deprecating ADR) + +## Context + +What is the context of this ADR? What is the issue that we are seeing? What is motivating this decision or change? + +## Decision + +What is the change we are proposing? What do we plan on doing to solve the issue? + +## Consequences + +What are the consequences of the change? What will be more difficult? What will be easier? \ No newline at end of file From c420ca3dea6f35b16f74ace9221302f95f7c155e Mon Sep 17 00:00:00 2001 From: Thomas Weber Date: Thu, 23 Mar 2023 15:26:10 +0100 Subject: [PATCH 10/28] init workflow test --- .github/workflows/main.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..d8487bf --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,17 @@ +# main.yml +on: [push] + +jobs: + hello_world_job: + runs-on: ubuntu-latest + name: A job to say hello + steps: + - name: Operator-SDK K8s Action + uses: dramirez-qb/operator-sdk-github-action@v3 + - uses: actions/checkout@v3 + - id: foo + uses: actions/hello-world-composite-action@v1 + with: + who-to-greet: 'Mona the Octocat' + - run: echo random-number ${{ steps.foo.outputs.random-number }} + shell: bash From afa4d532e70c6bf1e1047bc4643240affc4a790f Mon Sep 17 00:00:00 2001 From: Thomas Weber Date: Thu, 23 Mar 2023 15:52:56 +0100 Subject: [PATCH 11/28] test workflow --- .github/workflows/main.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d8487bf..28f6b5a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -9,9 +9,3 @@ jobs: - name: Operator-SDK K8s Action uses: dramirez-qb/operator-sdk-github-action@v3 - uses: actions/checkout@v3 - - id: foo - uses: actions/hello-world-composite-action@v1 - with: - who-to-greet: 'Mona the Octocat' - - run: echo random-number ${{ steps.foo.outputs.random-number }} - shell: bash From d3a7ea4c925604e94a74b94ce91024b439e6d678 Mon Sep 17 00:00:00 2001 From: Thomas Weber Date: Thu, 23 Mar 2023 16:08:26 +0100 Subject: [PATCH 12/28] test operator workflow --- .github/actions/operator.yml | 23 +++++++++++++++++++++++ .github/workflows/main.yml | 6 ++++-- 2 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 .github/actions/operator.yml diff --git a/.github/actions/operator.yml b/.github/actions/operator.yml new file mode 100644 index 0000000..2782361 --- /dev/null +++ b/.github/actions/operator.yml @@ -0,0 +1,23 @@ +name: 'Build Operator' +description: 'Build Operator' +inputs: + image: + description: 'Docker Image Name' + required: true + default: 'execDAT-operator' + tag: + description: 'Docker Image Tag' + required: false + default: 0.0.1 + dirPath: + description: 'Dockerfile Path' + required: false + default: 'execDAT-operator/Dockerfile' +runs: + using: 'composite' + steps: + - uses: dxas90/operator-sdk-github-action + with: + image: '${{ inputs.image }}' + tag: '${{ inputs.tag }}' + dirPath: '${{ inputs.dirPath }}' diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 28f6b5a..34d9cef 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,6 +6,8 @@ jobs: runs-on: ubuntu-latest name: A job to say hello steps: - - name: Operator-SDK K8s Action - uses: dramirez-qb/operator-sdk-github-action@v3 - uses: actions/checkout@v3 + with: + submodules: recursive + - uses: .github/actions/operator.yml + From d7ed5eb03eec6b4131af597a18891a73ec413e23 Mon Sep 17 00:00:00 2001 From: Thomas Weber Date: Thu, 23 Mar 2023 16:13:26 +0100 Subject: [PATCH 13/28] use local action --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 34d9cef..6b458c5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -9,5 +9,5 @@ jobs: - uses: actions/checkout@v3 with: submodules: recursive - - uses: .github/actions/operator.yml + - uses: ./actions/operator.yml From fcbc6b81194ccf6248660d64e22c858077869398 Mon Sep 17 00:00:00 2001 From: Thomas Weber Date: Thu, 23 Mar 2023 16:24:37 +0100 Subject: [PATCH 14/28] do not use recursive submodules --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6b458c5..6f28db9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,6 +8,6 @@ jobs: steps: - uses: actions/checkout@v3 with: - submodules: recursive + submodules: true - uses: ./actions/operator.yml From 95ba013aeaf4ef5d66e0b43bfcec9370d6cacc59 Mon Sep 17 00:00:00 2001 From: Thomas Weber Date: Thu, 23 Mar 2023 16:28:36 +0100 Subject: [PATCH 15/28] rename jobs --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6f28db9..d4dc89b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,9 +2,9 @@ on: [push] jobs: - hello_world_job: + operator: runs-on: ubuntu-latest - name: A job to say hello + name: build operator steps: - uses: actions/checkout@v3 with: From 8166f3c1bc7420f923e14f0cfbd49e0f50215dac Mon Sep 17 00:00:00 2001 From: Thomas Weber Date: Thu, 23 Mar 2023 16:32:12 +0100 Subject: [PATCH 16/28] use operator action --- .github/actions/operator.yml | 23 ----------------------- .github/workflows/main.yml | 5 ++--- execDAT-operator | 2 +- 3 files changed, 3 insertions(+), 27 deletions(-) delete mode 100644 .github/actions/operator.yml diff --git a/.github/actions/operator.yml b/.github/actions/operator.yml deleted file mode 100644 index 2782361..0000000 --- a/.github/actions/operator.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: 'Build Operator' -description: 'Build Operator' -inputs: - image: - description: 'Docker Image Name' - required: true - default: 'execDAT-operator' - tag: - description: 'Docker Image Tag' - required: false - default: 0.0.1 - dirPath: - description: 'Dockerfile Path' - required: false - default: 'execDAT-operator/Dockerfile' -runs: - using: 'composite' - steps: - - uses: dxas90/operator-sdk-github-action - with: - image: '${{ inputs.image }}' - tag: '${{ inputs.tag }}' - dirPath: '${{ inputs.dirPath }}' diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d4dc89b..48834cb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,6 +8,5 @@ jobs: steps: - uses: actions/checkout@v3 with: - submodules: true - - uses: ./actions/operator.yml - + submodules: recursive + - uses: ./exexDAT-operator/.github diff --git a/execDAT-operator b/execDAT-operator index 5b80818..72fd89a 160000 --- a/execDAT-operator +++ b/execDAT-operator @@ -1 +1 @@ -Subproject commit 5b8081847bbfcc88e90dc41c4417f6fe32ae52c7 +Subproject commit 72fd89a4a83942e5d1e2e33ba9864a23b885cbff From 867507ecfc2a0485e2a00645d4d7e79ad46d46db Mon Sep 17 00:00:00 2001 From: Thomas Weber Date: Thu, 23 Mar 2023 16:35:10 +0100 Subject: [PATCH 17/28] test action --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 48834cb..b1bf63d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -9,4 +9,6 @@ jobs: - uses: actions/checkout@v3 with: submodules: recursive - - uses: ./exexDAT-operator/.github + - run: ls -lah + - run: ls -lah ./execDAT-operator + - uses: ./execDAT-operator/.github From 4676b56ac671f799a4314b68d4b12a42e1d920ee Mon Sep 17 00:00:00 2001 From: Thomas Weber Date: Thu, 23 Mar 2023 16:37:54 +0100 Subject: [PATCH 18/28] just use path --- .github/workflows/main.yml | 4 +--- execDAT-operator | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b1bf63d..97d994d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -9,6 +9,4 @@ jobs: - uses: actions/checkout@v3 with: submodules: recursive - - run: ls -lah - - run: ls -lah ./execDAT-operator - - uses: ./execDAT-operator/.github + - uses: execDAT-operator/.github diff --git a/execDAT-operator b/execDAT-operator index 72fd89a..689fc4f 160000 --- a/execDAT-operator +++ b/execDAT-operator @@ -1 +1 @@ -Subproject commit 72fd89a4a83942e5d1e2e33ba9864a23b885cbff +Subproject commit 689fc4f858caf040ac6f04c1c63d7f949d84093f From 831aabe2c7b10c4d605fbd95ad93e179195ab5a7 Mon Sep 17 00:00:00 2001 From: Thomas Weber Date: Thu, 23 Mar 2023 16:38:24 +0100 Subject: [PATCH 19/28] use local path again --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 97d994d..ce85c07 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -9,4 +9,4 @@ jobs: - uses: actions/checkout@v3 with: submodules: recursive - - uses: execDAT-operator/.github + - uses: ./execDAT-operator/.github From 335149fe06fabe7dc30d6eb160dfb237e4794078 Mon Sep 17 00:00:00 2001 From: Thomas Weber Date: Thu, 23 Mar 2023 16:40:23 +0100 Subject: [PATCH 20/28] update submodule --- execDAT-operator | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/execDAT-operator b/execDAT-operator index 689fc4f..31e26e0 160000 --- a/execDAT-operator +++ b/execDAT-operator @@ -1 +1 @@ -Subproject commit 689fc4f858caf040ac6f04c1c63d7f949d84093f +Subproject commit 31e26e0bb31ffecbba52e582193df5be3f039df6 From 03a782b05838f7c2169554b91475bd90a929d41b Mon Sep 17 00:00:00 2001 From: Thomas Weber Date: Thu, 23 Mar 2023 16:44:17 +0100 Subject: [PATCH 21/28] update submodule --- execDAT-operator | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/execDAT-operator b/execDAT-operator index 31e26e0..2a6641e 160000 --- a/execDAT-operator +++ b/execDAT-operator @@ -1 +1 @@ -Subproject commit 31e26e0bb31ffecbba52e582193df5be3f039df6 +Subproject commit 2a6641e919b5f4c98876871d3eff42d54aacdbc3 From 60d02fe588cb448b2377c10bc3fb4c2924259d9b Mon Sep 17 00:00:00 2001 From: Thomas Weber Date: Thu, 23 Mar 2023 16:47:07 +0100 Subject: [PATCH 22/28] trigger submodule --- execDAT-operator | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/execDAT-operator b/execDAT-operator index 2a6641e..222af0e 160000 --- a/execDAT-operator +++ b/execDAT-operator @@ -1 +1 @@ -Subproject commit 2a6641e919b5f4c98876871d3eff42d54aacdbc3 +Subproject commit 222af0eaac8ce568038a59c1decf73af50ecd274 From 10bde2742a2b9717020d239762f17763ef5e2ef9 Mon Sep 17 00:00:00 2001 From: Thomas Weber Date: Thu, 23 Mar 2023 16:53:29 +0100 Subject: [PATCH 23/28] bump sm --- execDAT-operator | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/execDAT-operator b/execDAT-operator index 222af0e..ebebd3a 160000 --- a/execDAT-operator +++ b/execDAT-operator @@ -1 +1 @@ -Subproject commit 222af0eaac8ce568038a59c1decf73af50ecd274 +Subproject commit ebebd3a4cb94927cd17ae3ffabbedc1e7005c5a0 From f62e9a10b1e1d3faf853d05bb86c36ee44755742 Mon Sep 17 00:00:00 2001 From: Thomas Weber Date: Thu, 23 Mar 2023 17:06:42 +0100 Subject: [PATCH 24/28] bump sm --- execDAT-operator | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/execDAT-operator b/execDAT-operator index ebebd3a..8f876e9 160000 --- a/execDAT-operator +++ b/execDAT-operator @@ -1 +1 @@ -Subproject commit ebebd3a4cb94927cd17ae3ffabbedc1e7005c5a0 +Subproject commit 8f876e9dbca00950aa77593b517b7364dadeb63f From 5a7b26149fb30f0836a3ad98f26b482eb4a73d88 Mon Sep 17 00:00:00 2001 From: Thomas Weber Date: Thu, 23 Mar 2023 17:09:42 +0100 Subject: [PATCH 25/28] bump sm --- execDAT-operator | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/execDAT-operator b/execDAT-operator index 8f876e9..a5f14e4 160000 --- a/execDAT-operator +++ b/execDAT-operator @@ -1 +1 @@ -Subproject commit 8f876e9dbca00950aa77593b517b7364dadeb63f +Subproject commit a5f14e4d99599b0d3b905f5375e8c58bc9319a5c From 94904edb8a6ccc9159a3c48dea6cedff5d8bc3f7 Mon Sep 17 00:00:00 2001 From: Thomas Weber Date: Thu, 23 Mar 2023 17:13:47 +0100 Subject: [PATCH 26/28] bump sm --- execDAT-operator | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/execDAT-operator b/execDAT-operator index a5f14e4..7d98e59 160000 --- a/execDAT-operator +++ b/execDAT-operator @@ -1 +1 @@ -Subproject commit a5f14e4d99599b0d3b905f5375e8c58bc9319a5c +Subproject commit 7d98e5941944d992e05d68344e4b45d9675bd341 From c5a5a3ae96539892d39dccf4194543f21ff066c1 Mon Sep 17 00:00:00 2001 From: Thomas Weber Date: Thu, 23 Mar 2023 17:15:50 +0100 Subject: [PATCH 27/28] bump sm --- execDAT-operator | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/execDAT-operator b/execDAT-operator index 7d98e59..68e90e3 160000 --- a/execDAT-operator +++ b/execDAT-operator @@ -1 +1 @@ -Subproject commit 7d98e5941944d992e05d68344e4b45d9675bd341 +Subproject commit 68e90e33923a3457c8a4d224216f08b204def8f7 From e1082482955296d7481cddf32f4ee8d9c5bc6541 Mon Sep 17 00:00:00 2001 From: Thomas Weber Date: Sun, 4 Jun 2023 23:31:38 +0200 Subject: [PATCH 28/28] update k3d --- k3d-dev.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/k3d-dev.yaml b/k3d-dev.yaml index bd73196..9cb8e60 100644 --- a/k3d-dev.yaml +++ b/k3d-dev.yaml @@ -1,11 +1,11 @@ --- -apiVersion: k3d.io/v1alpha4 +apiVersion: k3d.io/v1alpha5 kind: Simple metadata: name: execdat-dev servers: 1 agents: 0 -image: docker.io/rancher/k3s:v1.25.6-k3s1 +image: docker.io/rancher/k3s:v1.26.5-k3s1 # kubeAPI: # same as `--api-port myhost.my.domain:6445` (where the name would resolve to 127.0.0.1) # host: "myhost.my.domain" # important for the `server` setting in the kubeconfig @@ -42,6 +42,14 @@ options: - arg: "--tls-san=k3d.localhost" nodeFilters: - server:* + - arg: "--kubelet-arg=eviction-hard=imagefs.available<1%,nodefs.available<1%" + nodeFilters: + - server:* + - agents:* + - arg: "--kubelet-arg=eviction-minimum-reclaim=imagefs.available=1%,nodefs.available=1%" + nodeFilters: + - server:* + - agents:* kubeconfig: updateDefaultKubeconfig: true # add new cluster to your default Kubeconfig; same as `--kubeconfig-update-default` (default: true) switchCurrentContext: true # also set current-context to the new cluster's context; same as `--kubeconfig-switch-context` (default: true)