From 4409c61013e1db3338125850ee78ff96cd17cfa1 Mon Sep 17 00:00:00 2001 From: Ori Shadmon Date: Wed, 29 Apr 2026 17:49:21 -0700 Subject: [PATCH 01/46] version control Signed-off-by: Ori Shadmon --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 10a9cdbf..c5c52c8d 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ $(info LOADING MAKEFILE) # ────────────────────────────────────────────── export IS_MANUAL ?= false export ANYLOG_TYPE ?= anylog-generic -export TAG ?= pre-develop +export TAG ?= 1.4.2604 export IMAGE ?= anylogco/anylog-network export TEST_CONN ?= From 2aa2e74cd4aa998b0f8db3ea551758455c007d9d Mon Sep 17 00:00:00 2001 From: Ori Shadmon Date: Thu, 30 Apr 2026 14:16:23 -0700 Subject: [PATCH 02/46] tmp Signed-off-by: Ori Shadmon --- deploy.sh | 2 +- docker-makefiles/anylog-standalone-publisher/node_configs.env | 4 ++-- docker-makefiles/build_docker_compose.sh | 2 +- docker-makefiles/prep_configs.sh | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/deploy.sh b/deploy.sh index e54d0b8e..f035d978 100644 --- a/deploy.sh +++ b/deploy.sh @@ -2,7 +2,7 @@ # deploy.sh — AnyLog node lifecycle manager # Usage: bash deploy.sh [OPTIONS] # Run: bash deploy.sh help -set -euo pipefail +#set -euo pipefail # ────────────────────────────────────────────── # Defaults (override via environment or flags) diff --git a/docker-makefiles/anylog-standalone-publisher/node_configs.env b/docker-makefiles/anylog-standalone-publisher/node_configs.env index 78003efb..7bd339c7 100644 --- a/docker-makefiles/anylog-standalone-publisher/node_configs.env +++ b/docker-makefiles/anylog-standalone-publisher/node_configs.env @@ -14,7 +14,7 @@ IMAGE=anylogco/anylog-network # the repo if needed OR a local path for deployment-scripts and then use that instead of cloning into the repositroy DEPLOYMENTS_REPO=https://github.com/AnyLog-co/deployment-scripts # Branch associated with git repo -DEPLOYMENTS_BRANCH="pre-develop" +DEPLOYMENTS_BRANCH="os-dev" # specify the docker socket - to be used when deploying docker monitoring DOCKER_SOCKET="/var/run/docker.sock" @@ -279,7 +279,7 @@ BUCKET_SECRET_KEY="" #==== remote-gui ==== # Companion UI application - deployed alongside the AnyLog node # Whether to enable Remote-GUI in deployment -ENABLE_REMOTE_GUI=true +ENABLE_REMOTE_GUI=false # Remote-GUI version Tag REMOTE_GUI_TAG=beta2 # NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 diff --git a/docker-makefiles/build_docker_compose.sh b/docker-makefiles/build_docker_compose.sh index 939739bc..757e3c70 100644 --- a/docker-makefiles/build_docker_compose.sh +++ b/docker-makefiles/build_docker_compose.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -euo pipefail +#set -euo pipefail # -------- Helpers -------- die() { diff --git a/docker-makefiles/prep_configs.sh b/docker-makefiles/prep_configs.sh index b13fba3f..30d88654 100644 --- a/docker-makefiles/prep_configs.sh +++ b/docker-makefiles/prep_configs.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -euo pipefail +#set -euo pipefail # -------- Helpers -------- die() { From d399b0706874a208fda3a80d2188ac52b1c62f30 Mon Sep 17 00:00:00 2001 From: Ori Shadmon Date: Thu, 30 Apr 2026 14:17:17 -0700 Subject: [PATCH 03/46] revert os-dev code Signed-off-by: Ori Shadmon --- .../anylog-generic/node_configs.env | 2 +- .../anylog-master/node_configs.env | 2 +- .../anylog-operator/node_configs.env | 2 +- .../anylog-publisher/node_configs.env | 2 +- .../anylog-query/node_configs.env | 2 +- .../node_configs.env | 2 +- .../node_configs.env | 6 +- docker-makefiles/build_docker_compose.sh | 2 +- docker-makefiles/master/node_configs.env | 289 +++++++++++++++++ docker-makefiles/operator1/node_configs.env | 289 +++++++++++++++++ docker-makefiles/operator2/node_configs.env | 289 +++++++++++++++++ docker-makefiles/operator3/node_configs.env | 289 +++++++++++++++++ docker-makefiles/operator4/node_configs.env | 289 +++++++++++++++++ docker-makefiles/prep_configs.sh | 2 +- docker-makefiles/roy-master/node_configs.env | 296 ++++++++++++++++++ .../roy-operator1/node_configs.env | 296 ++++++++++++++++++ openbao/docker-compose.yml | 24 ++ openbao/get4bao.sh | 6 + openbao/openbao.env | 9 + openbao/openbao_cmds.py | 270 ++++++++++++++++ openbao/run_openbao.sh | 7 + row_count.sh | 33 ++ 22 files changed, 2397 insertions(+), 11 deletions(-) create mode 100644 docker-makefiles/master/node_configs.env create mode 100644 docker-makefiles/operator1/node_configs.env create mode 100644 docker-makefiles/operator2/node_configs.env create mode 100644 docker-makefiles/operator3/node_configs.env create mode 100644 docker-makefiles/operator4/node_configs.env create mode 100644 docker-makefiles/roy-master/node_configs.env create mode 100644 docker-makefiles/roy-operator1/node_configs.env create mode 100644 openbao/docker-compose.yml create mode 100644 openbao/get4bao.sh create mode 100644 openbao/openbao.env create mode 100644 openbao/openbao_cmds.py create mode 100644 openbao/run_openbao.sh create mode 100644 row_count.sh diff --git a/docker-makefiles/anylog-generic/node_configs.env b/docker-makefiles/anylog-generic/node_configs.env index 87b68ed1..2af268ca 100644 --- a/docker-makefiles/anylog-generic/node_configs.env +++ b/docker-makefiles/anylog-generic/node_configs.env @@ -282,7 +282,7 @@ BUCKET_SECRET_KEY="" # Whether to enable Remote-GUI in deployment ENABLE_REMOTE_GUI=false # Remote-GUI version Tag -REMOTE_GUI_TAG=1.1.0 +REMOTE_GUI_TAG=beta2 # NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 REMOTE_GUI_NIC="" # Remote GUI Frontend Port diff --git a/docker-makefiles/anylog-master/node_configs.env b/docker-makefiles/anylog-master/node_configs.env index 86c7f1e3..f5dd3bcb 100644 --- a/docker-makefiles/anylog-master/node_configs.env +++ b/docker-makefiles/anylog-master/node_configs.env @@ -281,7 +281,7 @@ BUCKET_SECRET_KEY="" # Whether to enable Remote-GUI in deployment ENABLE_REMOTE_GUI=false # Remote-GUI version Tag -REMOTE_GUI_TAG=1.1.0 +REMOTE_GUI_TAG=beta2 # NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 REMOTE_GUI_NIC="" # Remote GUI Frontend Port diff --git a/docker-makefiles/anylog-operator/node_configs.env b/docker-makefiles/anylog-operator/node_configs.env index 28fa423e..1edc8566 100644 --- a/docker-makefiles/anylog-operator/node_configs.env +++ b/docker-makefiles/anylog-operator/node_configs.env @@ -282,7 +282,7 @@ BUCKET_SECRET_KEY="" # Whether to enable Remote-GUI in deployment ENABLE_REMOTE_GUI=false # Remote-GUI version Tag -REMOTE_GUI_TAG=1.1.0 +REMOTE_GUI_TAG=beta2 # NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 REMOTE_GUI_NIC="" # Remote GUI Frontend Port diff --git a/docker-makefiles/anylog-publisher/node_configs.env b/docker-makefiles/anylog-publisher/node_configs.env index 69971810..e49a862b 100644 --- a/docker-makefiles/anylog-publisher/node_configs.env +++ b/docker-makefiles/anylog-publisher/node_configs.env @@ -282,7 +282,7 @@ BUCKET_SECRET_KEY="" # Whether to enable Remote-GUI in deployment ENABLE_REMOTE_GUI=false # Remote-GUI version Tag -REMOTE_GUI_TAG=1.1.0 +REMOTE_GUI_TAG=beta2 # NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 REMOTE_GUI_NIC="" # Remote GUI Frontend Port diff --git a/docker-makefiles/anylog-query/node_configs.env b/docker-makefiles/anylog-query/node_configs.env index 305e80c1..fc7bc875 100644 --- a/docker-makefiles/anylog-query/node_configs.env +++ b/docker-makefiles/anylog-query/node_configs.env @@ -282,7 +282,7 @@ BUCKET_SECRET_KEY="" # Whether to enable Remote-GUI in deployment ENABLE_REMOTE_GUI=true # Remote-GUI version Tag -REMOTE_GUI_TAG=1.1.0 +REMOTE_GUI_TAG=beta2 # NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 REMOTE_GUI_NIC="" # Remote GUI Frontend Port diff --git a/docker-makefiles/anylog-standalone-operator/node_configs.env b/docker-makefiles/anylog-standalone-operator/node_configs.env index 1b3e743d..7ccfc082 100644 --- a/docker-makefiles/anylog-standalone-operator/node_configs.env +++ b/docker-makefiles/anylog-standalone-operator/node_configs.env @@ -281,7 +281,7 @@ BUCKET_SECRET_KEY="" # Whether to enable Remote-GUI in deployment ENABLE_REMOTE_GUI=false # Remote-GUI version Tag -REMOTE_GUI_TAG=1.1.0 +REMOTE_GUI_TAG=beta2 # NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 REMOTE_GUI_NIC="" # Remote GUI Frontend Port diff --git a/docker-makefiles/anylog-standalone-publisher/node_configs.env b/docker-makefiles/anylog-standalone-publisher/node_configs.env index b429c58c..78003efb 100644 --- a/docker-makefiles/anylog-standalone-publisher/node_configs.env +++ b/docker-makefiles/anylog-standalone-publisher/node_configs.env @@ -14,7 +14,7 @@ IMAGE=anylogco/anylog-network # the repo if needed OR a local path for deployment-scripts and then use that instead of cloning into the repositroy DEPLOYMENTS_REPO=https://github.com/AnyLog-co/deployment-scripts # Branch associated with git repo -DEPLOYMENTS_BRANCH="os-dev" +DEPLOYMENTS_BRANCH="pre-develop" # specify the docker socket - to be used when deploying docker monitoring DOCKER_SOCKET="/var/run/docker.sock" @@ -279,9 +279,9 @@ BUCKET_SECRET_KEY="" #==== remote-gui ==== # Companion UI application - deployed alongside the AnyLog node # Whether to enable Remote-GUI in deployment -ENABLE_REMOTE_GUI=false +ENABLE_REMOTE_GUI=true # Remote-GUI version Tag -REMOTE_GUI_TAG=1.1.0 +REMOTE_GUI_TAG=beta2 # NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 REMOTE_GUI_NIC="" # Remote GUI Frontend Port diff --git a/docker-makefiles/build_docker_compose.sh b/docker-makefiles/build_docker_compose.sh index 757e3c70..939739bc 100644 --- a/docker-makefiles/build_docker_compose.sh +++ b/docker-makefiles/build_docker_compose.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -#set -euo pipefail +set -euo pipefail # -------- Helpers -------- die() { diff --git a/docker-makefiles/master/node_configs.env b/docker-makefiles/master/node_configs.env new file mode 100644 index 00000000..4b232fae --- /dev/null +++ b/docker-makefiles/master/node_configs.env @@ -0,0 +1,289 @@ +#==== .env ==== +# Basic deployment settings + +# Container image, deployment type +# -> prod: run AnyLog instance +# -> bash: run shell interface with AnyLog installed and params set +INIT_TYPE=prod + +# AnyLog image +IMAGE=anylogco/anylog-network + +# Github repo to use for configuring / deploying node +DEPLOYMENTS_REPO="https://github.com/AnyLog-co/deployment-scripts" +# Branch associated with git repo +DEPLOYMENTS_BRANCH="pre-develop" + +#==== basic ==== +# Core node configuration - required for all deployments +#--- General --- +# Configuration based deploymnet node deployment type +# Node Types: +# generic: sandbox with only network configured +# master: "blockchain" emulator +# operator: agent dedicated to storing data +# query: agent dedicated to querying data (enable system_query logical database) +# publisher: agent used to distribute data +# master-operator: a combination of master and operator on a single agent +# master-publisher: a combination of master and publisher on a single agent +NODE_TYPE=master + +# Name of the AnyLog instance +NODE_NAME=master +# Owner of the AnyLog instance +COMPANY_NAME=AnyLog Co. + +#--- Networking --- +# Port address used by AnyLog's TCP protocol to communicate with other nodes in the network +ANYLOG_SERVER_PORT=32048 +# Port address used by AnyLog's REST protocol +ANYLOG_REST_PORT=32049 +# Port value to be used as an MQTT broker, or some other third-party broker +ANYLOG_BROKER_PORT="" + +#--- Database --- +# Physical database type (sqlite, psql) +DB_TYPE=sqlite +# Database IP address +DB_IP=127.0.0.1 +# Database port number +DB_PORT=5432 +# Whether to set autocommit data +AUTOCOMMIT=true + +#--- NoSQL --- +# Enable external blob storage +BLOBS_STORAGE=false +# Storage backend type - Supported: mongo, s3, akave, minio +BLOB_STORAGE_TYPE=mongo +# Storage endpoint (MongoDB or object storage endpoint) +BLOB_STORAGE_IP=127.0.0.1 +# Storage port (used mainly for MongoDB) +BLOB_STORAGE_PORT=27017 + +#--- System Query --- +# Whether to start system_query logical database +SYSTEM_QUERY=false +# Database type used for system_query logical database +SYSTEM_QUERY_DB=sqlite +# Run system_query using in-memory SQLite. If set to false, will use pre-set database type +MEMORY=false + +#--- Blockchain --- +# TCP connection information for Master Node +LEDGER_CONN=127.0.0.1:32048 +# How often to sync from blockchain +BLOCKCHAIN_SYNC=60 second +# Source of where the blockchain data is coming from (master, optimism) +BLOCKCHAIN_SOURCE=master +# Where the local copy of the blockchain will be stored +BLOCKCHAIN_DESTINATION=file + +#--- Operator --- +# Owner of the cluster +CLUSTER_NAME=anylog-cluster1 +# Logical database name +DEFAULT_DBMS=mydb + +#==== southbound ==== +# Data ingestion protocols (MQTT, OPC-UA, video streaming, monitoring) +#--- Monitoring --- +# Whether to monitor the node or not +NODE_MONITORING=false +# Store monitoring in Operator node(s) +STORE_MONITORING=false +# Accept syslog data from local (Message broker required) +SYSLOG_MONITORING=false +# Docker Monitoring +DOCKER_MONITORING=false + +#--- Video Streaming --- +# Enable or disable video streaming. +# true = stream video, false = do not stream +ENABLE_VIDEO_STREAMING=false +# URL of the video to stream. Can be a YouTube link or other video source. +VIDEO_URL=https://www.youtube.com/watch?v=rnXIjl_Rzy4 +# Port on which the video streaming server will run. +VIDEO_PORT=32800 +# Name of the video stream or service identifier. +VIDEO_NAME=sfstream +# Enable or disable object detections (like YOLO detections) on the video. +# true = enable detections, false = disable +ENABLE_DETECTIONS=false +# Port on which the YOLO model server will run for object detection. +YOLO_MODEL_PORT=50051 + +#--- MQTT --- +# Whether to enable the default MQTT process +ENABLE_MQTT=false + +# IP address of MQTT broker +MQTT_BROKER=172.104.228.251 +# Port associated with MQTT broker +MQTT_PORT=1883 +# Whether to enable MQTT logging process +MQTT_LOG=false + +# Topic to get data for +MSG_TOPIC=anylog-demo +# Logical database name +MSG_DBMS=mydb +# Table where to store data +MSG_TABLE=bring [table] +# Timestamp column name +MSG_TIMESTAMP_COLUMN=bring [timestamp] +# Value column name +MSG_VALUE_COLUMN=bring [value] +# Column value type +MSG_VALUE_COLUMN_TYPE=float + +#--- OPC-UA --- +# Whether or not to enable to OPC-UA service +ENABLE_OPCUA=false +# OPC-UA URL address (ex. opcua.tcp:;//127.0.0.1:4840) +OPCUA_URL="" +# Node information the root is located in (ex. ns=2;s=DataSet) +OPCUA_NODE="" +# How often to pull data from OPC-UA +OPCUA_FREQUENCY=5 + +#==== advanced ==== +# Fine-tuning for experienced users - safe to leave as defaults +#--- Directories --- +# AnyLog Root Path +ANYLOG_PATH=/app +# !local_scripts: ${ANYLOG_HOME}/deployment-scripts/scripts +LOCAL_SCRIPTS=/app/deployment-scripts +# !test_dir: ${ANYLOG_HOME}/deployment-scripts/test +TEST_DIR=/app/deployment-scripts/test + +#--- General --- +# hide blockchain policy - supports all node types except cluster / operator +IS_HIDDEN=false +# Whether to automatically run a local (or personalized) script at the end of the process +DEPLOY_LOCAL_SCRIPT=false +# Coordinates of the machine - used by Grafana to map the network +LOCATION="" +# Country where machine is located +COUNTRY="" +# State where machine is located +STATE="" +# City where machine is located +CITY="" + +#--- Networking --- +# Declare Policy name +CONFIG_NAME="" +# Whether to enable DNS resolution +ENABLE_DNS=false +# NIC type used to resolve the internal IP address (ex. eth0, lo) +NIC_TYPE="" +# Overlay IP address - if set, will replace local IP address when connecting to network +OVERLAY_IP="192.168.0.140" +# External DNS resolution +EXTERNAL_DNS="" +# local DNS domain to append to hostname (ex. example.com) +DNS_DOMAIN="" +# Full path for machine's (local) DNS +DNS="" + +# Bind settings - a true value binds to a specific IP and Port (a false value binds to all IPs) +TCP_BIND=false +REST_BIND=false +BROKER_BIND=false + +# Number of concurrent threads per connection protocol +TCP_THREADS=6 +REST_THREADS=6 +BROKER_THREADS=6 + +# Timeout in seconds for REST requests (0 = continuous) +REST_TIMEOUT=30 + +#--- NoSQL --- +# Reuse identical blobs using file hash +BLOBS_REUSE=true +# Compress blobs before storage +BLOBS_COMPRESS=true + +# Folder for storing blobs locally +# Leave empty to use default application path +BLOBS_FOLDER="" + +#--- Object Store --- +# Logical bucket grouping +BUCKET_GROUP="" +# Bucket identifier +BUCKET_ID="" +# Cloud region +BUCKET_REGION="" + +#--- Operator --- +# Operator ID - autogenerated INT if not specified +# This is used to help keep a record of where data was originally stored +MEMBER="" +# Whether of not to enable HA against the cluster +ENABLE_HA=false +# How many days back to sync between nodes +START_DATE=30 +# How many threads to use in the operator process +OPERATOR_THREADS=3 + +#--- Sub processes --- +# Compress JSON and SQL file(s) backup +COMPRESS_FILE=true +# When data comes in write to database immediately, as opposed to waiting for a full buffer +WRITE_IMMEDIATE=true +# If buffer is not full, how long to wait until pushing data through +THRESHOLD_TIME=60 seconds +# Buffer size to reach, at which point data is pushed through +THRESHOLD_VOLUME=10KB +# Number of parallel queries +QUERY_POOL=6 + +#==== secrets ==== +# Sensitive credentials - do not commit to version control +# will be updated to use sercets logic for both docker and k8s + +#--- General --- +# AnyLog License Key +LICENSE_KEY=5528032c1ee582225942bae2631c9adc97213543c9caec371e55c70c5a7c4ed2303cef87ab33e6499a42daf52cfb41a09b7f558334acf08c42d72f122f1cba88164ebeba48f176e2b261b7b8b34d0d404cd7397df7b80b16812706a01f632c6dc6ee27d1b69f39884b68c938242088cc60852103651c092f82e7ba52f4f41430{"company":"Roy","expiration":"2036-01-30","type":"beta"} + +#--- Database --- +# Username for SQL database connection +DB_USER=admin +# Password correlated to database user +DB_PASSWD=passwd + +#--- NoSQL --- +# Optional DB authentication +BLOB_STORAGE_USER=admin +BLOB_STORAGE_PASSWORD=passwd + +#--- MQTT --- +# User associated with MQTT broker +MQTT_USER=anyloguser +# Password associated with MQTT user +MQTT_PASSWD=mqtt4AnyLog! + +#--- Object Store --- +# Access credentials +BUCKET_ACCESS_KEY="" +BUCKET_SECRET_KEY="" + +#==== remote-gui ==== +# Companion UI application - deployed alongside the AnyLog node +# Whether to enable Remote-GUI in deployment +ENABLE_REMOTE_GUI=false +# Remote-GUI version Tag +REMOTE_GUI_TAG=beta2 +# NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 +REMOTE_GUI_NIC="" +# Remote GUI Frontend Port +REMOTE_GUI_FE=31800 +# Remote GUI Backend Port +REMOTE_GUI_BE=8080 +# Grafana PATH to reference for Grafana connection +GRAFANA_URL="" + + diff --git a/docker-makefiles/operator1/node_configs.env b/docker-makefiles/operator1/node_configs.env new file mode 100644 index 00000000..887792bd --- /dev/null +++ b/docker-makefiles/operator1/node_configs.env @@ -0,0 +1,289 @@ +#==== .env ==== +# Basic deployment settings + +# Container image, deployment type +# -> prod: run AnyLog instance +# -> bash: run shell interface with AnyLog installed and params set +INIT_TYPE=prod + +# AnyLog image +IMAGE=anylogco/anylog-network + +# Github repo to use for configuring / deploying node +DEPLOYMENTS_REPO="https://github.com/AnyLog-co/deployment-scripts" +# Branch associated with git repo +DEPLOYMENTS_BRANCH="pre-develop" + +#==== basic ==== +# Core node configuration - required for all deployments +#--- General --- +# Configuration based deploymnet node deployment type +# Node Types: +# generic: sandbox with only network configured +# master: "blockchain" emulator +# operator: agent dedicated to storing data +# query: agent dedicated to querying data (enable system_query logical database) +# publisher: agent used to distribute data +# master-operator: a combination of master and operator on a single agent +# master-publisher: a combination of master and publisher on a single agent +NODE_TYPE=operator + +# Name of the AnyLog instance +NODE_NAME=operator1 +# Owner of the AnyLog instance +COMPANY_NAME=AnyLog Co. + +#--- Networking --- +# Port address used by AnyLog's TCP protocol to communicate with other nodes in the network +ANYLOG_SERVER_PORT=32148 +# Port address used by AnyLog's REST protocol +ANYLOG_REST_PORT=32149 +# Port value to be used as an MQTT broker, or some other third-party broker +ANYLOG_BROKER_PORT=32150 + +#--- Database --- +# Physical database type (sqlite, psql) +DB_TYPE=sqlite +# Database IP address +DB_IP=${INET_IP} +# Database port number +DB_PORT=5432 +# Whether to set autocommit data +AUTOCOMMIT=true + +#--- NoSQL --- +# Enable external blob storage +BLOBS_STORAGE=false +# Storage backend type - Supported: mongo, s3, akave, minio +BLOB_STORAGE_TYPE=mongo +# Storage endpoint (MongoDB or object storage endpoint) +BLOB_STORAGE_IP=${INET_IP} +# Storage port (used mainly for MongoDB) +BLOB_STORAGE_PORT=27017 + +#--- System Query --- +# Whether to start system_query logical database +SYSTEM_QUERY=True +# Database type used for system_query logical database +SYSTEM_QUERY_DB=sqlite +# Run system_query using in-memory SQLite. If set to false, will use pre-set database type +MEMORY=True + +#--- Blockchain --- +# TCP connection information for Master Node +LEDGER_CONN=${INET_IP}:32048 +# How often to sync from blockchain +BLOCKCHAIN_SYNC=60 second +# Source of where the blockchain data is coming from (master, optimism) +BLOCKCHAIN_SOURCE=master +# Where the local copy of the blockchain will be stored +BLOCKCHAIN_DESTINATION=file + +#--- Operator --- +# Owner of the cluster +CLUSTER_NAME=anylog-cluster1 +# Logical database name +DEFAULT_DBMS=mydb + +#==== southbound ==== +# Data ingestion protocols (MQTT, OPC-UA, video streaming, monitoring) +#--- Monitoring --- +# Whether to monitor the node or not +NODE_MONITORING=false +# Store monitoring in Operator node(s) +STORE_MONITORING=false +# Accept syslog data from local (Message broker required) +SYSLOG_MONITORING=false +# Docker Monitoring +DOCKER_MONITORING=false + +#--- Video Streaming --- +# Enable or disable video streaming. +# true = stream video, false = do not stream +ENABLE_VIDEO_STREAMING=false +# URL of the video to stream. Can be a YouTube link or other video source. +VIDEO_URL=https://www.youtube.com/watch?v=rnXIjl_Rzy4 +# Port on which the video streaming server will run. +VIDEO_PORT=32800 +# Name of the video stream or service identifier. +VIDEO_NAME=sfstream +# Enable or disable object detections (like YOLO detections) on the video. +# true = enable detections, false = disable +ENABLE_DETECTIONS=false +# Port on which the YOLO model server will run for object detection. +YOLO_MODEL_PORT=50051 + +#--- MQTT --- +# Whether to enable the default MQTT process +ENABLE_MQTT=false + +# IP address of MQTT broker +MQTT_BROKER=172.104.228.251 +# Port associated with MQTT broker +MQTT_PORT=1883 +# Whether to enable MQTT logging process +MQTT_LOG=false + +# Topic to get data for +MSG_TOPIC=anylog-demo +# Logical database name +MSG_DBMS=mydb +# Table where to store data +MSG_TABLE=bring [table] +# Timestamp column name +MSG_TIMESTAMP_COLUMN=bring [timestamp] +# Value column name +MSG_VALUE_COLUMN=bring [value] +# Column value type +MSG_VALUE_COLUMN_TYPE=float + +#--- OPC-UA --- +# Whether or not to enable to OPC-UA service +ENABLE_OPCUA=false +# OPC-UA URL address (ex. opcua.tcp:;//127.0.0.1:4840) +OPCUA_URL="" +# Node information the root is located in (ex. ns=2;s=DataSet) +OPCUA_NODE="" +# How often to pull data from OPC-UA +OPCUA_FREQUENCY=5 + +#==== advanced ==== +# Fine-tuning for experienced users - safe to leave as defaults +#--- Directories --- +# AnyLog Root Path +ANYLOG_PATH=/app +# !local_scripts: ${ANYLOG_HOME}/deployment-scripts/scripts +LOCAL_SCRIPTS=/app/deployment-scripts +# !test_dir: ${ANYLOG_HOME}/deployment-scripts/test +TEST_DIR=/app/deployment-scripts/test + +#--- General --- +# hide blockchain policy - supports all node types except cluster / operator +IS_HIDDEN=false +# Whether to automatically run a local (or personalized) script at the end of the process +DEPLOY_LOCAL_SCRIPT=false +# Coordinates of the machine - used by Grafana to map the network +LOCATION="" +# Country where machine is located +COUNTRY="" +# State where machine is located +STATE="" +# City where machine is located +CITY="" + +#--- Networking --- +# Declare Policy name +CONFIG_NAME="" +# Whether to enable DNS resolution +ENABLE_DNS=false +# NIC type used to resolve the internal IP address (ex. eth0, lo) +NIC_TYPE="" +# Overlay IP address - if set, will replace local IP address when connecting to network +OVERLAY_IP="192.168.0.140" +# External DNS resolution +EXTERNAL_DNS="" +# local DNS domain to append to hostname (ex. example.com) +DNS_DOMAIN="" +# Full path for machine's (local) DNS +DNS="" + +# Bind settings - a true value binds to a specific IP and Port (a false value binds to all IPs) +TCP_BIND=false +REST_BIND=false +BROKER_BIND=false + +# Number of concurrent threads per connection protocol +TCP_THREADS=6 +REST_THREADS=6 +BROKER_THREADS=6 + +# Timeout in seconds for REST requests (0 = continuous) +REST_TIMEOUT=30 + +#--- NoSQL --- +# Reuse identical blobs using file hash +BLOBS_REUSE=true +# Compress blobs before storage +BLOBS_COMPRESS=true + +# Folder for storing blobs locally +# Leave empty to use default application path +BLOBS_FOLDER="" + +#--- Object Store --- +# Logical bucket grouping +BUCKET_GROUP="" +# Bucket identifier +BUCKET_ID="" +# Cloud region +BUCKET_REGION="" + +#--- Operator --- +# Operator ID - autogenerated INT if not specified +# This is used to help keep a record of where data was originally stored +MEMBER="" +# Whether of not to enable HA against the cluster +ENABLE_HA=false +# How many days back to sync between nodes +START_DATE=30 +# How many threads to use in the operator process +OPERATOR_THREADS=3 + +#--- Sub processes --- +# Compress JSON and SQL file(s) backup +COMPRESS_FILE=true +# When data comes in write to database immediately, as opposed to waiting for a full buffer +WRITE_IMMEDIATE=true +# If buffer is not full, how long to wait until pushing data through +THRESHOLD_TIME=1 seconds +# Buffer size to reach, at which point data is pushed through +THRESHOLD_VOLUME=10KB +# Number of parallel queries +QUERY_POOL=6 + +#==== secrets ==== +# Sensitive credentials - do not commit to version control +# will be updated to use sercets logic for both docker and k8s + +#--- General --- +# AnyLog License Key +LICENSE_KEY=5528032c1ee582225942bae2631c9adc97213543c9caec371e55c70c5a7c4ed2303cef87ab33e6499a42daf52cfb41a09b7f558334acf08c42d72f122f1cba88164ebeba48f176e2b261b7b8b34d0d404cd7397df7b80b16812706a01f632c6dc6ee27d1b69f39884b68c938242088cc60852103651c092f82e7ba52f4f41430{"company":"Roy","expiration":"2036-01-30","type":"beta"} + +#--- Database --- +# Username for SQL database connection +DB_USER=demo +# Password correlated to database user +DB_PASSWD=passwd + +#--- NoSQL --- +# Optional DB authentication +BLOB_STORAGE_USER=admin +BLOB_STORAGE_PASSWORD=passwd + +#--- MQTT --- +# User associated with MQTT broker +MQTT_USER=anyloguser +# Password associated with MQTT user +MQTT_PASSWD=mqtt4AnyLog! + +#--- Object Store --- +# Access credentials +BUCKET_ACCESS_KEY="" +BUCKET_SECRET_KEY="" + +#==== remote-gui ==== +# Companion UI application - deployed alongside the AnyLog node +# Whether to enable Remote-GUI in deployment +ENABLE_REMOTE_GUI=false +# Remote-GUI version Tag +REMOTE_GUI_TAG=beta2 +# NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 +REMOTE_GUI_NIC="" +# Remote GUI Frontend Port +REMOTE_GUI_FE=31800 +# Remote GUI Backend Port +REMOTE_GUI_BE=8080 +# Grafana PATH to reference for Grafana connection +GRAFANA_URL="" + + diff --git a/docker-makefiles/operator2/node_configs.env b/docker-makefiles/operator2/node_configs.env new file mode 100644 index 00000000..918df85c --- /dev/null +++ b/docker-makefiles/operator2/node_configs.env @@ -0,0 +1,289 @@ +#==== .env ==== +# Basic deployment settings + +# Container image, deployment type +# -> prod: run AnyLog instance +# -> bash: run shell interface with AnyLog installed and params set +INIT_TYPE=prod + +# AnyLog image +IMAGE=anylogco/anylog-network + +# Github repo to use for configuring / deploying node +DEPLOYMENTS_REPO="https://github.com/AnyLog-co/deployment-scripts" +# Branch associated with git repo +DEPLOYMENTS_BRANCH="pre-develop" + +#==== basic ==== +# Core node configuration - required for all deployments +#--- General --- +# Configuration based deploymnet node deployment type +# Node Types: +# generic: sandbox with only network configured +# master: "blockchain" emulator +# operator: agent dedicated to storing data +# query: agent dedicated to querying data (enable system_query logical database) +# publisher: agent used to distribute data +# master-operator: a combination of master and operator on a single agent +# master-publisher: a combination of master and publisher on a single agent +NODE_TYPE=operator + +# Name of the AnyLog instance +NODE_NAME=operator2 +# Owner of the AnyLog instance +COMPANY_NAME=AnyLog Co. + +#--- Networking --- +# Port address used by AnyLog's TCP protocol to communicate with other nodes in the network +ANYLOG_SERVER_PORT=32248 +# Port address used by AnyLog's REST protocol +ANYLOG_REST_PORT=32249 +# Port value to be used as an MQTT broker, or some other third-party broker +ANYLOG_BROKER_PORT=32250 + +#--- Database --- +# Physical database type (sqlite, psql) +DB_TYPE=sqlite +# Database IP address +DB_IP=${INET_IP} +# Database port number +DB_PORT=5433 +# Whether to set autocommit data +AUTOCOMMIT=true + +#--- NoSQL --- +# Enable external blob storage +BLOBS_STORAGE=false +# Storage backend type - Supported: mongo, s3, akave, minio +BLOB_STORAGE_TYPE=mongo +# Storage endpoint (MongoDB or object storage endpoint) +BLOB_STORAGE_IP=${INET_IP} +# Storage port (used mainly for MongoDB) +BLOB_STORAGE_PORT=27017 + +#--- System Query --- +# Whether to start system_query logical database +SYSTEM_QUERY=true +# Database type used for system_query logical database +SYSTEM_QUERY_DB=sqlite +# Run system_query using in-memory SQLite. If set to false, will use pre-set database type +MEMORY=true + +#--- Blockchain --- +# TCP connection information for Master Node +LEDGER_CONN=${INET_IP}:32048 +# How often to sync from blockchain +BLOCKCHAIN_SYNC=10 second +# Source of where the blockchain data is coming from (master, optimism) +BLOCKCHAIN_SOURCE=master +# Where the local copy of the blockchain will be stored +BLOCKCHAIN_DESTINATION=file + +#--- Operator --- +# Owner of the cluster +CLUSTER_NAME=anylog-cluster2 +# Logical database name +DEFAULT_DBMS=mydb + +#==== southbound ==== +# Data ingestion protocols (MQTT, OPC-UA, video streaming, monitoring) +#--- Monitoring --- +# Whether to monitor the node or not +NODE_MONITORING=false +# Store monitoring in Operator node(s) +STORE_MONITORING=false +# Accept syslog data from local (Message broker required) +SYSLOG_MONITORING=false +# Docker Monitoring +DOCKER_MONITORING=false + +#--- Video Streaming --- +# Enable or disable video streaming. +# true = stream video, false = do not stream +ENABLE_VIDEO_STREAMING=false +# URL of the video to stream. Can be a YouTube link or other video source. +VIDEO_URL=https://www.youtube.com/watch?v=rnXIjl_Rzy4 +# Port on which the video streaming server will run. +VIDEO_PORT=32800 +# Name of the video stream or service identifier. +VIDEO_NAME=sfstream +# Enable or disable object detections (like YOLO detections) on the video. +# true = enable detections, false = disable +ENABLE_DETECTIONS=false +# Port on which the YOLO model server will run for object detection. +YOLO_MODEL_PORT=50051 + +#--- MQTT --- +# Whether to enable the default MQTT process +ENABLE_MQTT=false + +# IP address of MQTT broker +MQTT_BROKER=172.104.228.251 +# Port associated with MQTT broker +MQTT_PORT=1883 +# Whether to enable MQTT logging process +MQTT_LOG=false + +# Topic to get data for +MSG_TOPIC=anylog-demo +# Logical database name +MSG_DBMS=mydb +# Table where to store data +MSG_TABLE=bring [table] +# Timestamp column name +MSG_TIMESTAMP_COLUMN=bring [timestamp] +# Value column name +MSG_VALUE_COLUMN=bring [value] +# Column value type +MSG_VALUE_COLUMN_TYPE=float + +#--- OPC-UA --- +# Whether or not to enable to OPC-UA service +ENABLE_OPCUA=false +# OPC-UA URL address (ex. opcua.tcp:;//127.0.0.1:4840) +OPCUA_URL="" +# Node information the root is located in (ex. ns=2;s=DataSet) +OPCUA_NODE="" +# How often to pull data from OPC-UA +OPCUA_FREQUENCY=5 + +#==== advanced ==== +# Fine-tuning for experienced users - safe to leave as defaults +#--- Directories --- +# AnyLog Root Path +ANYLOG_PATH=/app +# !local_scripts: ${ANYLOG_HOME}/deployment-scripts/scripts +LOCAL_SCRIPTS=/app/deployment-scripts +# !test_dir: ${ANYLOG_HOME}/deployment-scripts/test +TEST_DIR=/app/deployment-scripts/test + +#--- General --- +# hide blockchain policy - supports all node types except cluster / operator +IS_HIDDEN=false +# Whether to automatically run a local (or personalized) script at the end of the process +DEPLOY_LOCAL_SCRIPT=false +# Coordinates of the machine - used by Grafana to map the network +LOCATION="" +# Country where machine is located +COUNTRY="" +# State where machine is located +STATE="" +# City where machine is located +CITY="" + +#--- Networking --- +# Declare Policy name +CONFIG_NAME="" +# Whether to enable DNS resolution +ENABLE_DNS=false +# NIC type used to resolve the internal IP address (ex. eth0, lo) +NIC_TYPE="" +# Overlay IP address - if set, will replace local IP address when connecting to network +OVERLAY_IP="192.168.0.140" +# External DNS resolution +EXTERNAL_DNS="" +# local DNS domain to append to hostname (ex. example.com) +DNS_DOMAIN="" +# Full path for machine's (local) DNS +DNS="" + +# Bind settings - a true value binds to a specific IP and Port (a false value binds to all IPs) +TCP_BIND=false +REST_BIND=false +BROKER_BIND=false + +# Number of concurrent threads per connection protocol +TCP_THREADS=6 +REST_THREADS=6 +BROKER_THREADS=6 + +# Timeout in seconds for REST requests (0 = continuous) +REST_TIMEOUT=30 + +#--- NoSQL --- +# Reuse identical blobs using file hash +BLOBS_REUSE=true +# Compress blobs before storage +BLOBS_COMPRESS=true + +# Folder for storing blobs locally +# Leave empty to use default application path +BLOBS_FOLDER="" + +#--- Object Store --- +# Logical bucket grouping +BUCKET_GROUP="" +# Bucket identifier +BUCKET_ID="" +# Cloud region +BUCKET_REGION="" + +#--- Operator --- +# Operator ID - autogenerated INT if not specified +# This is used to help keep a record of where data was originally stored +MEMBER="" +# Whether of not to enable HA against the cluster +ENABLE_HA=false +# How many days back to sync between nodes +START_DATE=30 +# How many threads to use in the operator process +OPERATOR_THREADS=3 + +#--- Sub processes --- +# Compress JSON and SQL file(s) backup +COMPRESS_FILE=true +# When data comes in write to database immediately, as opposed to waiting for a full buffer +WRITE_IMMEDIATE=true +# If buffer is not full, how long to wait until pushing data through +THRESHOLD_TIME=60 seconds +# Buffer size to reach, at which point data is pushed through +THRESHOLD_VOLUME=10KB +# Number of parallel queries +QUERY_POOL=6 + +#==== secrets ==== +# Sensitive credentials - do not commit to version control +# will be updated to use sercets logic for both docker and k8s + +#--- General --- +# AnyLog License Key +LICENSE_KEY=5528032c1ee582225942bae2631c9adc97213543c9caec371e55c70c5a7c4ed2303cef87ab33e6499a42daf52cfb41a09b7f558334acf08c42d72f122f1cba88164ebeba48f176e2b261b7b8b34d0d404cd7397df7b80b16812706a01f632c6dc6ee27d1b69f39884b68c938242088cc60852103651c092f82e7ba52f4f41430{"company":"Roy","expiration":"2036-01-30","type":"beta"} + +#--- Database --- +# Username for SQL database connection +DB_USER=demo +# Password correlated to database user +DB_PASSWD=passwd + +#--- NoSQL --- +# Optional DB authentication +BLOB_STORAGE_USER=admin +BLOB_STORAGE_PASSWORD=passwd + +#--- MQTT --- +# User associated with MQTT broker +MQTT_USER=anyloguser +# Password associated with MQTT user +MQTT_PASSWD=mqtt4AnyLog! + +#--- Object Store --- +# Access credentials +BUCKET_ACCESS_KEY="" +BUCKET_SECRET_KEY="" + +#==== remote-gui ==== +# Companion UI application - deployed alongside the AnyLog node +# Whether to enable Remote-GUI in deployment +ENABLE_REMOTE_GUI=false +# Remote-GUI version Tag +REMOTE_GUI_TAG=beta2 +# NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 +REMOTE_GUI_NIC="" +# Remote GUI Frontend Port +REMOTE_GUI_FE=31800 +# Remote GUI Backend Port +REMOTE_GUI_BE=8080 +# Grafana PATH to reference for Grafana connection +GRAFANA_URL="" + + diff --git a/docker-makefiles/operator3/node_configs.env b/docker-makefiles/operator3/node_configs.env new file mode 100644 index 00000000..e11b7358 --- /dev/null +++ b/docker-makefiles/operator3/node_configs.env @@ -0,0 +1,289 @@ +#==== .env ==== +# Basic deployment settings + +# Container image, deployment type +# -> prod: run AnyLog instance +# -> bash: run shell interface with AnyLog installed and params set +INIT_TYPE=prod + +# AnyLog image +IMAGE=anylogco/anylog-network + +# Github repo to use for configuring / deploying node +DEPLOYMENTS_REPO="https://github.com/AnyLog-co/deployment-scripts" +# Branch associated with git repo +DEPLOYMENTS_BRANCH="pre-develop" + +#==== basic ==== +# Core node configuration - required for all deployments +#--- General --- +# Configuration based deploymnet node deployment type +# Node Types: +# generic: sandbox with only network configured +# master: "blockchain" emulator +# operator: agent dedicated to storing data +# query: agent dedicated to querying data (enable system_query logical database) +# publisher: agent used to distribute data +# master-operator: a combination of master and operator on a single agent +# master-publisher: a combination of master and publisher on a single agent +NODE_TYPE=operator + +# Name of the AnyLog instance +NODE_NAME=operator3 +# Owner of the AnyLog instance +COMPANY_NAME=AnyLog Co. + +#--- Networking --- +# Port address used by AnyLog's TCP protocol to communicate with other nodes in the network +ANYLOG_SERVER_PORT=32348 +# Port address used by AnyLog's REST protocol +ANYLOG_REST_PORT=32349 +# Port value to be used as an MQTT broker, or some other third-party broker +ANYLOG_BROKER_PORT=32350 + +#--- Database --- +# Physical database type (sqlite, psql) +DB_TYPE=sqlite +# Database IP address +DB_IP=${INET_IP} +# Database port number +DB_PORT=5434 +# Whether to set autocommit data +AUTOCOMMIT=true + +#--- NoSQL --- +# Enable external blob storage +BLOBS_STORAGE=false +# Storage backend type - Supported: mongo, s3, akave, minio +BLOB_STORAGE_TYPE=mongo +# Storage endpoint (MongoDB or object storage endpoint) +BLOB_STORAGE_IP=${INET_IP} +# Storage port (used mainly for MongoDB) +BLOB_STORAGE_PORT=27017 + +#--- System Query --- +# Whether to start system_query logical database +SYSTEM_QUERY=true +# Database type used for system_query logical database +SYSTEM_QUERY_DB=sqlite +# Run system_query using in-memory SQLite. If set to false, will use pre-set database type +MEMORY=true + +#--- Blockchain --- +# TCP connection information for Master Node +LEDGER_CONN=${INET_IP}:32048 +# How often to sync from blockchain +BLOCKCHAIN_SYNC=10 second +# Source of where the blockchain data is coming from (master, optimism) +BLOCKCHAIN_SOURCE=master +# Where the local copy of the blockchain will be stored +BLOCKCHAIN_DESTINATION=file + +#--- Operator --- +# Owner of the cluster +CLUSTER_NAME=anylog-cluster3 +# Logical database name +DEFAULT_DBMS=mydb + +#==== southbound ==== +# Data ingestion protocols (MQTT, OPC-UA, video streaming, monitoring) +#--- Monitoring --- +# Whether to monitor the node or not +NODE_MONITORING=false +# Store monitoring in Operator node(s) +STORE_MONITORING=false +# Accept syslog data from local (Message broker required) +SYSLOG_MONITORING=false +# Docker Monitoring +DOCKER_MONITORING=false + +#--- Video Streaming --- +# Enable or disable video streaming. +# true = stream video, false = do not stream +ENABLE_VIDEO_STREAMING=false +# URL of the video to stream. Can be a YouTube link or other video source. +VIDEO_URL=https://www.youtube.com/watch?v=rnXIjl_Rzy4 +# Port on which the video streaming server will run. +VIDEO_PORT=32800 +# Name of the video stream or service identifier. +VIDEO_NAME=sfstream +# Enable or disable object detections (like YOLO detections) on the video. +# true = enable detections, false = disable +ENABLE_DETECTIONS=false +# Port on which the YOLO model server will run for object detection. +YOLO_MODEL_PORT=50051 + +#--- MQTT --- +# Whether to enable the default MQTT process +ENABLE_MQTT=false + +# IP address of MQTT broker +MQTT_BROKER=172.104.228.251 +# Port associated with MQTT broker +MQTT_PORT=1883 +# Whether to enable MQTT logging process +MQTT_LOG=false + +# Topic to get data for +MSG_TOPIC=anylog-demo +# Logical database name +MSG_DBMS=mydb +# Table where to store data +MSG_TABLE=bring [table] +# Timestamp column name +MSG_TIMESTAMP_COLUMN=bring [timestamp] +# Value column name +MSG_VALUE_COLUMN=bring [value] +# Column value type +MSG_VALUE_COLUMN_TYPE=float + +#--- OPC-UA --- +# Whether or not to enable to OPC-UA service +ENABLE_OPCUA=false +# OPC-UA URL address (ex. opcua.tcp:;//127.0.0.1:4840) +OPCUA_URL="" +# Node information the root is located in (ex. ns=2;s=DataSet) +OPCUA_NODE="" +# How often to pull data from OPC-UA +OPCUA_FREQUENCY=5 + +#==== advanced ==== +# Fine-tuning for experienced users - safe to leave as defaults +#--- Directories --- +# AnyLog Root Path +ANYLOG_PATH=/app +# !local_scripts: ${ANYLOG_HOME}/deployment-scripts/scripts +LOCAL_SCRIPTS=/app/deployment-scripts +# !test_dir: ${ANYLOG_HOME}/deployment-scripts/test +TEST_DIR=/app/deployment-scripts/test + +#--- General --- +# hide blockchain policy - supports all node types except cluster / operator +IS_HIDDEN=false +# Whether to automatically run a local (or personalized) script at the end of the process +DEPLOY_LOCAL_SCRIPT=false +# Coordinates of the machine - used by Grafana to map the network +LOCATION="" +# Country where machine is located +COUNTRY="" +# State where machine is located +STATE="" +# City where machine is located +CITY="" + +#--- Networking --- +# Declare Policy name +CONFIG_NAME="" +# Whether to enable DNS resolution +ENABLE_DNS=false +# NIC type used to resolve the internal IP address (ex. eth0, lo) +NIC_TYPE="" +# Overlay IP address - if set, will replace local IP address when connecting to network +OVERLAY_IP="192.168.0.140" +# External DNS resolution +EXTERNAL_DNS="" +# local DNS domain to append to hostname (ex. example.com) +DNS_DOMAIN="" +# Full path for machine's (local) DNS +DNS="" + +# Bind settings - a true value binds to a specific IP and Port (a false value binds to all IPs) +TCP_BIND=false +REST_BIND=false +BROKER_BIND=false + +# Number of concurrent threads per connection protocol +TCP_THREADS=6 +REST_THREADS=6 +BROKER_THREADS=6 + +# Timeout in seconds for REST requests (0 = continuous) +REST_TIMEOUT=30 + +#--- NoSQL --- +# Reuse identical blobs using file hash +BLOBS_REUSE=true +# Compress blobs before storage +BLOBS_COMPRESS=true + +# Folder for storing blobs locally +# Leave empty to use default application path +BLOBS_FOLDER="" + +#--- Object Store --- +# Logical bucket grouping +BUCKET_GROUP="" +# Bucket identifier +BUCKET_ID="" +# Cloud region +BUCKET_REGION="" + +#--- Operator --- +# Operator ID - autogenerated INT if not specified +# This is used to help keep a record of where data was originally stored +MEMBER="" +# Whether of not to enable HA against the cluster +ENABLE_HA=false +# How many days back to sync between nodes +START_DATE=30 +# How many threads to use in the operator process +OPERATOR_THREADS=3 + +#--- Sub processes --- +# Compress JSON and SQL file(s) backup +COMPRESS_FILE=true +# When data comes in write to database immediately, as opposed to waiting for a full buffer +WRITE_IMMEDIATE=true +# If buffer is not full, how long to wait until pushing data through +THRESHOLD_TIME=10 seconds +# Buffer size to reach, at which point data is pushed through +THRESHOLD_VOLUME=10KB +# Number of parallel queries +QUERY_POOL=6 + +#==== secrets ==== +# Sensitive credentials - do not commit to version control +# will be updated to use sercets logic for both docker and k8s + +#--- General --- +# AnyLog License Key +LICENSE_KEY=5528032c1ee582225942bae2631c9adc97213543c9caec371e55c70c5a7c4ed2303cef87ab33e6499a42daf52cfb41a09b7f558334acf08c42d72f122f1cba88164ebeba48f176e2b261b7b8b34d0d404cd7397df7b80b16812706a01f632c6dc6ee27d1b69f39884b68c938242088cc60852103651c092f82e7ba52f4f41430{"company":"Roy","expiration":"2036-01-30","type":"beta"} + +#--- Database --- +# Username for SQL database connection +DB_USER=demo +# Password correlated to database user +DB_PASSWD=passwd + +#--- NoSQL --- +# Optional DB authentication +BLOB_STORAGE_USER=admin +BLOB_STORAGE_PASSWORD=passwd + +#--- MQTT --- +# User associated with MQTT broker +MQTT_USER=anyloguser +# Password associated with MQTT user +MQTT_PASSWD=mqtt4AnyLog! + +#--- Object Store --- +# Access credentials +BUCKET_ACCESS_KEY="" +BUCKET_SECRET_KEY="" + +#==== remote-gui ==== +# Companion UI application - deployed alongside the AnyLog node +# Whether to enable Remote-GUI in deployment +ENABLE_REMOTE_GUI=false +# Remote-GUI version Tag +REMOTE_GUI_TAG=beta2 +# NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 +REMOTE_GUI_NIC="" +# Remote GUI Frontend Port +REMOTE_GUI_FE=31800 +# Remote GUI Backend Port +REMOTE_GUI_BE=8080 +# Grafana PATH to reference for Grafana connection +GRAFANA_URL="" + + diff --git a/docker-makefiles/operator4/node_configs.env b/docker-makefiles/operator4/node_configs.env new file mode 100644 index 00000000..72a39895 --- /dev/null +++ b/docker-makefiles/operator4/node_configs.env @@ -0,0 +1,289 @@ +#==== .env ==== +# Basic deployment settings + +# Container image, deployment type +# -> prod: run AnyLog instance +# -> bash: run shell interface with AnyLog installed and params set +INIT_TYPE=prod + +# AnyLog image +IMAGE=anylogco/anylog-network + +# Github repo to use for configuring / deploying node +DEPLOYMENTS_REPO="https://github.com/AnyLog-co/deployment-scripts" +# Branch associated with git repo +DEPLOYMENTS_BRANCH="pre-develop" + +#==== basic ==== +# Core node configuration - required for all deployments +#--- General --- +# Configuration based deploymnet node deployment type +# Node Types: +# generic: sandbox with only network configured +# master: "blockchain" emulator +# operator: agent dedicated to storing data +# query: agent dedicated to querying data (enable system_query logical database) +# publisher: agent used to distribute data +# master-operator: a combination of master and operator on a single agent +# master-publisher: a combination of master and publisher on a single agent +NODE_TYPE=operator + +# Name of the AnyLog instance +NODE_NAME=operator4 +# Owner of the AnyLog instance +COMPANY_NAME=AnyLog Co. + +#--- Networking --- +# Port address used by AnyLog's TCP protocol to communicate with other nodes in the network +ANYLOG_SERVER_PORT=32548 +# Port address used by AnyLog's REST protocol +ANYLOG_REST_PORT=32549 +# Port value to be used as an MQTT broker, or some other third-party broker +ANYLOG_BROKER_PORT=32550 + +#--- Database --- +# Physical database type (sqlite, psql) +DB_TYPE=sqlite +# Database IP address +DB_IP=${INET_IP} +# Database port number +DB_PORT=5435 +# Whether to set autocommit data +AUTOCOMMIT=true + +#--- NoSQL --- +# Enable external blob storage +BLOBS_STORAGE=false +# Storage backend type - Supported: mongo, s3, akave, minio +BLOB_STORAGE_TYPE=mongo +# Storage endpoint (MongoDB or object storage endpoint) +BLOB_STORAGE_IP=${INET_IP} +# Storage port (used mainly for MongoDB) +BLOB_STORAGE_PORT=27017 + +#--- System Query --- +# Whether to start system_query logical database +SYSTEM_QUERY=true +# Database type used for system_query logical database +SYSTEM_QUERY_DB=sqlite +# Run system_query using in-memory SQLite. If set to false, will use pre-set database type +MEMORY=true + +#--- Blockchain --- +# TCP connection information for Master Node +LEDGER_CONN=${INET_IP}:32048 +# How often to sync from blockchain +BLOCKCHAIN_SYNC=10 second +# Source of where the blockchain data is coming from (master, optimism) +BLOCKCHAIN_SOURCE=master +# Where the local copy of the blockchain will be stored +BLOCKCHAIN_DESTINATION=file + +#--- Operator --- +# Owner of the cluster +CLUSTER_NAME=anylog-cluster4 +# Logical database name +DEFAULT_DBMS=mydb + +#==== southbound ==== +# Data ingestion protocols (MQTT, OPC-UA, video streaming, monitoring) +#--- Monitoring --- +# Whether to monitor the node or not +NODE_MONITORING=false +# Store monitoring in Operator node(s) +STORE_MONITORING=false +# Accept syslog data from local (Message broker required) +SYSLOG_MONITORING=false +# Docker Monitoring +DOCKER_MONITORING=false + +#--- Video Streaming --- +# Enable or disable video streaming. +# true = stream video, false = do not stream +ENABLE_VIDEO_STREAMING=false +# URL of the video to stream. Can be a YouTube link or other video source. +VIDEO_URL=https://www.youtube.com/watch?v=rnXIjl_Rzy4 +# Port on which the video streaming server will run. +VIDEO_PORT=32800 +# Name of the video stream or service identifier. +VIDEO_NAME=sfstream +# Enable or disable object detections (like YOLO detections) on the video. +# true = enable detections, false = disable +ENABLE_DETECTIONS=false +# Port on which the YOLO model server will run for object detection. +YOLO_MODEL_PORT=50051 + +#--- MQTT --- +# Whether to enable the default MQTT process +ENABLE_MQTT=false + +# IP address of MQTT broker +MQTT_BROKER=172.104.228.251 +# Port associated with MQTT broker +MQTT_PORT=1883 +# Whether to enable MQTT logging process +MQTT_LOG=false + +# Topic to get data for +MSG_TOPIC=anylog-demo +# Logical database name +MSG_DBMS=mydb +# Table where to store data +MSG_TABLE=bring [table] +# Timestamp column name +MSG_TIMESTAMP_COLUMN=bring [timestamp] +# Value column name +MSG_VALUE_COLUMN=bring [value] +# Column value type +MSG_VALUE_COLUMN_TYPE=float + +#--- OPC-UA --- +# Whether or not to enable to OPC-UA service +ENABLE_OPCUA=false +# OPC-UA URL address (ex. opcua.tcp:;//127.0.0.1:4840) +OPCUA_URL="" +# Node information the root is located in (ex. ns=2;s=DataSet) +OPCUA_NODE="" +# How often to pull data from OPC-UA +OPCUA_FREQUENCY=5 + +#==== advanced ==== +# Fine-tuning for experienced users - safe to leave as defaults +#--- Directories --- +# AnyLog Root Path +ANYLOG_PATH=/app +# !local_scripts: ${ANYLOG_HOME}/deployment-scripts/scripts +LOCAL_SCRIPTS=/app/deployment-scripts +# !test_dir: ${ANYLOG_HOME}/deployment-scripts/test +TEST_DIR=/app/deployment-scripts/test + +#--- General --- +# hide blockchain policy - supports all node types except cluster / operator +IS_HIDDEN=false +# Whether to automatically run a local (or personalized) script at the end of the process +DEPLOY_LOCAL_SCRIPT=false +# Coordinates of the machine - used by Grafana to map the network +LOCATION="" +# Country where machine is located +COUNTRY="" +# State where machine is located +STATE="" +# City where machine is located +CITY="" + +#--- Networking --- +# Declare Policy name +CONFIG_NAME="" +# Whether to enable DNS resolution +ENABLE_DNS=false +# NIC type used to resolve the internal IP address (ex. eth0, lo) +NIC_TYPE="" +# Overlay IP address - if set, will replace local IP address when connecting to network +OVERLAY_IP="" +# External DNS resolution +EXTERNAL_DNS="" +# local DNS domain to append to hostname (ex. example.com) +DNS_DOMAIN="" +# Full path for machine's (local) DNS +DNS="" + +# Bind settings - a true value binds to a specific IP and Port (a false value binds to all IPs) +TCP_BIND=false +REST_BIND=false +BROKER_BIND=false + +# Number of concurrent threads per connection protocol +TCP_THREADS=6 +REST_THREADS=6 +BROKER_THREADS=6 + +# Timeout in seconds for REST requests (0 = continuous) +REST_TIMEOUT=30 + +#--- NoSQL --- +# Reuse identical blobs using file hash +BLOBS_REUSE=true +# Compress blobs before storage +BLOBS_COMPRESS=true + +# Folder for storing blobs locally +# Leave empty to use default application path +BLOBS_FOLDER="" + +#--- Object Store --- +# Logical bucket grouping +BUCKET_GROUP="" +# Bucket identifier +BUCKET_ID="" +# Cloud region +BUCKET_REGION="" + +#--- Operator --- +# Operator ID - autogenerated INT if not specified +# This is used to help keep a record of where data was originally stored +MEMBER="" +# Whether of not to enable HA against the cluster +ENABLE_HA=false +# How many days back to sync between nodes +START_DATE=30 +# How many threads to use in the operator process +OPERATOR_THREADS=3 + +#--- Sub processes --- +# Compress JSON and SQL file(s) backup +COMPRESS_FILE=true +# When data comes in write to database immediately, as opposed to waiting for a full buffer +WRITE_IMMEDIATE=true +# If buffer is not full, how long to wait until pushing data through +THRESHOLD_TIME=10 seconds +# Buffer size to reach, at which point data is pushed through +THRESHOLD_VOLUME=10KB +# Number of parallel queries +QUERY_POOL=6 + +#==== secrets ==== +# Sensitive credentials - do not commit to version control +# will be updated to use sercets logic for both docker and k8s + +#--- General --- +# AnyLog License Key +LICENSE_KEY=5528032c1ee582225942bae2631c9adc97213543c9caec371e55c70c5a7c4ed2303cef87ab33e6499a42daf52cfb41a09b7f558334acf08c42d72f122f1cba88164ebeba48f176e2b261b7b8b34d0d404cd7397df7b80b16812706a01f632c6dc6ee27d1b69f39884b68c938242088cc60852103651c092f82e7ba52f4f41430{"company":"Roy","expiration":"2036-01-30","type":"beta"} + +#--- Database --- +# Username for SQL database connection +DB_USER=demo +# Password correlated to database user +DB_PASSWD=passwd + +#--- NoSQL --- +# Optional DB authentication +BLOB_STORAGE_USER=admin +BLOB_STORAGE_PASSWORD=passwd + +#--- MQTT --- +# User associated with MQTT broker +MQTT_USER=anyloguser +# Password associated with MQTT user +MQTT_PASSWD=mqtt4AnyLog! + +#--- Object Store --- +# Access credentials +BUCKET_ACCESS_KEY="" +BUCKET_SECRET_KEY="" + +#==== remote-gui ==== +# Companion UI application - deployed alongside the AnyLog node +# Whether to enable Remote-GUI in deployment +ENABLE_REMOTE_GUI=false +# Remote-GUI version Tag +REMOTE_GUI_TAG=beta2 +# NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 +REMOTE_GUI_NIC="" +# Remote GUI Frontend Port +REMOTE_GUI_FE=31800 +# Remote GUI Backend Port +REMOTE_GUI_BE=8080 +# Grafana PATH to reference for Grafana connection +GRAFANA_URL="" + + diff --git a/docker-makefiles/prep_configs.sh b/docker-makefiles/prep_configs.sh index 30d88654..b13fba3f 100644 --- a/docker-makefiles/prep_configs.sh +++ b/docker-makefiles/prep_configs.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -#set -euo pipefail +set -euo pipefail # -------- Helpers -------- die() { diff --git a/docker-makefiles/roy-master/node_configs.env b/docker-makefiles/roy-master/node_configs.env new file mode 100644 index 00000000..5949d8ef --- /dev/null +++ b/docker-makefiles/roy-master/node_configs.env @@ -0,0 +1,296 @@ +#==== .env ==== +# Basic deployment settings + +# Container image, deployment type +# -> prod: run AnyLog instance +# -> bash: run shell interface with AnyLog installed and params set +INIT_TYPE=prod + +# AnyLog image +IMAGE=anylogco/anylog-network + +# Github repo to use for configuring / deploying node +DEPLOYMENTS_REPO="/mnt/c/Users/oshad/AnyLog-code/deployment-scripts" +# Branch associated with git repo +DEPLOYMENTS_BRANCH="os-dev" + +# specify the docker socket - to be used when deploying docker monitoring +DOCKER_SOCKET="/var/run/docker.sock" + +#==== basic ==== +# Core node configuration - required for all deployments +#--- General --- +# Configuration based deploymnet node deployment type +# Node Types: +# generic: sandbox with only network configured +# master: "blockchain" emulator +# operator: agent dedicated to storing data +# query: agent dedicated to querying data (enable system_query logical database) +# publisher: agent used to distribute data +# master-operator: a combination of master and operator on a single agent +# master-publisher: a combination of master and publisher on a single agent +NODE_TYPE=master + +# Name of the AnyLog instance +NODE_NAME=master +# Owner of the AnyLog instance +COMPANY_NAME=AnyLog Co. + +#--- Networking --- +# Port address used by AnyLog's TCP protocol to communicate with other nodes in the network +ANYLOG_SERVER_PORT=32048 +# Port address used by AnyLog's REST protocol +ANYLOG_REST_PORT=32049 +# Port value to be used as an MQTT broker, or some other third-party broker +ANYLOG_BROKER_PORT="" + +#--- Database --- +# Physical database type (sqlite, psql) +DB_TYPE=sqlite +# Database IP address +DB_IP=127.0.0.1 +# Database port number +DB_PORT=5432 +# Whether to set autocommit data +AUTOCOMMIT=true + +#--- NoSQL --- +# Enable external blob storage +BLOBS_STORAGE=false +# Storage backend type - Supported: mongo, s3, akave, minio +BLOB_STORAGE_TYPE=mongo +# Storage endpoint (MongoDB or object storage endpoint) +BLOB_STORAGE_IP=127.0.0.1 +# Storage port (used mainly for MongoDB) +BLOB_STORAGE_PORT=27017 + +#--- System Query --- +# Whether to start system_query logical database +SYSTEM_QUERY=false +# Database type used for system_query logical database +SYSTEM_QUERY_DB=sqlite +# Run system_query using in-memory SQLite. If set to false, will use pre-set database type +MEMORY=false + +#--- Blockchain --- +# TCP connection information for Master Node +LEDGER_CONN=127.0.0.1:32048 +# How often to sync from blockchain +BLOCKCHAIN_SYNC=60 second +# Source of where the blockchain data is coming from (master, optimism) +BLOCKCHAIN_SOURCE=master +# Where the local copy of the blockchain will be stored +BLOCKCHAIN_DESTINATION=file + +#--- Operator --- +# Owner of the cluster +CLUSTER_NAME=anylog-cluster1 +# Logical database name +DEFAULT_DBMS=mydb + +#==== southbound ==== +# Data ingestion protocols (MQTT, OPC-UA, video streaming, monitoring) +#--- Monitoring --- +# Whether to monitor the node or not +NODE_MONITORING=false +# Store monitoring in Operator node(s) +STORE_MONITORING=false +# Accept syslog data from local (Message broker required) +SYSLOG_MONITORING=false +# Docker Monitoring +DOCKER_MONITORING=false + +#--- Video Streaming --- +# Enable or disable video streaming. +# true = stream video, false = do not stream +ENABLE_VIDEO_STREAMING=false +# URL of the video to stream. Can be a YouTube link or other video source. +VIDEO_URL=https://www.youtube.com/watch?v=rnXIjl_Rzy4 +# Port on which the video streaming server will run. +VIDEO_PORT=32800 +# Name of the video stream or service identifier. +VIDEO_NAME=sfstream +# Enable or disable object detections (like YOLO detections) on the video. +# true = enable detections, false = disable +ENABLE_DETECTIONS=false +# Port on which the YOLO model server will run for object detection. +YOLO_MODEL_PORT=50051 + +#--- MQTT --- +# Whether to enable the default MQTT process +ENABLE_MQTT=false + +# IP address of MQTT broker +MQTT_BROKER=172.104.228.251 +# Port associated with MQTT broker +MQTT_PORT=1883 +# Whether to enable MQTT logging process +MQTT_LOG=false + +# Topic to get data for +MSG_TOPIC=anylog-demo +# Logical database name +MSG_DBMS=mydb +# Table where to store data +MSG_TABLE=bring [table] +# Timestamp column name +MSG_TIMESTAMP_COLUMN=bring [timestamp] +# Value column name +MSG_VALUE_COLUMN=bring [value] +# Column value type +MSG_VALUE_COLUMN_TYPE=float + +#--- OPC-UA --- +# Whether or not to enable to OPC-UA service +ENABLE_OPCUA=false +# OPC-UA URL address (ex. opcua.tcp:;//127.0.0.1:4840) +OPCUA_URL="" +# Node information the root is located in (ex. ns=2;s=DataSet) +OPCUA_NODE="" +# How often to pull data from OPC-UA +OPCUA_FREQUENCY=5 + +#==== advanced ==== +# Fine-tuning for experienced users - safe to leave as defaults +#--- Directories --- +# AnyLog Root Path +ANYLOG_PATH=/app +# !local_scripts: ${ANYLOG_HOME}/deployment-scripts/scripts +LOCAL_SCRIPTS=/app/deployment-scripts +# !test_dir: ${ANYLOG_HOME}/deployment-scripts/test +TEST_DIR=/app/deployment-scripts/test + +#--- General --- +# hide blockchain policy - supports all node types except cluster / operator +IS_HIDDEN=false +# Whether to automatically run a local (or personalized) script at the end of the process +DEPLOY_LOCAL_SCRIPT=false +# Coordinates of the machine - used by Grafana to map the network +LOCATION="" +# Country where machine is located +COUNTRY="" +# State where machine is located +STATE="" +# City where machine is located +CITY="" + +#--- Networking --- +# Declare Policy name +CONFIG_NAME="" +# Whether to enable DNS resolution +ENABLE_DNS=false +# NIC type used to resolve the internal IP address (ex. eth0, lo) +NIC_TYPE="" +# Overlay IP address - if set, will replace local IP address when connecting to network +OVERLAY_IP="" +# External DNS resolution +EXTERNAL_DNS="" +# local DNS domain to append to hostname (ex. example.com) +DNS_DOMAIN="" +# Full path for machine's (local) DNS +DNS="" + +# Bind settings - a true value binds to a specific IP and Port (a false value binds to all IPs) +TCP_BIND=false +REST_BIND=false +BROKER_BIND=false + +# Number of concurrent threads per connection protocol +TCP_THREADS=6 +REST_THREADS=6 +BROKER_THREADS=6 + +# Timeout in seconds for REST requests (0 = continuous) +REST_TIMEOUT=30 + +#--- NoSQL --- +# Reuse identical blobs using file hash +BLOBS_REUSE=true +# Compress blobs before storage +BLOBS_COMPRESS=true + +# Folder for storing blobs locally +# Leave empty to use default application path +BLOBS_FOLDER="" + +#--- Object Store --- +# Logical bucket grouping +BUCKET_GROUP="" +# Bucket identifier +BUCKET_ID="" +# Cloud region +BUCKET_REGION="" + +#--- Operator --- +# Operator ID - autogenerated INT if not specified +# This is used to help keep a record of where data was originally stored +MEMBER="" +# Whether of not to enable HA against the cluster +ENABLE_HA=false +# How many days back to sync between nodes +START_DATE=30 +# How many threads to use in the operator process +OPERATOR_THREADS=3 + +#--- Sub processes --- +# Compress JSON and SQL file(s) backup +COMPRESS_FILE=true +# When data comes in write to database immediately, as opposed to waiting for a full buffer +WRITE_IMMEDIATE=true +# If buffer is not full, how long to wait until pushing data through +THRESHOLD_TIME=60 seconds +# Buffer size to reach, at which point data is pushed through +THRESHOLD_VOLUME=10KB +# Number of parallel queries +QUERY_POOL=6 + +#--- DEBUG --- +ENABLE_TRACEBACK=false +TRACE_LEVEL=0 + +#==== secrets ==== +# Sensitive credentials - do not commit to version control +# will be updated to use secrets logic for both docker and k8s + +#--- General --- +# AnyLog License Key +LICENSE_KEY=5528032c1ee582225942bae2631c9adc97213543c9caec371e55c70c5a7c4ed2303cef87ab33e6499a42daf52cfb41a09b7f558334acf08c42d72f122f1cba88164ebeba48f176e2b261b7b8b34d0d404cd7397df7b80b16812706a01f632c6dc6ee27d1b69f39884b68c938242088cc60852103651c092f82e7ba52f4f41430{"company":"Roy","expiration":"2036-01-30","type":"beta"} + +#--- Database --- +# Username for SQL database connection +DB_USER=admin +# Password correlated to database user +DB_PASSWD=passwd + +#--- NoSQL --- +# Optional DB authentication +BLOB_STORAGE_USER=admin +BLOB_STORAGE_PASSWORD=passwd + +#--- MQTT --- +# User associated with MQTT broker +MQTT_USER=anyloguser +# Password associated with MQTT user +MQTT_PASSWD=mqtt4AnyLog! + +#--- Object Store --- +# Access credentials +BUCKET_ACCESS_KEY="" +BUCKET_SECRET_KEY="" + +#==== remote-gui ==== +# Companion UI application - deployed alongside the AnyLog node +# Whether to enable Remote-GUI in deployment +ENABLE_REMOTE_GUI=false +# Remote-GUI version Tag +REMOTE_GUI_TAG=beta2 +# NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 +REMOTE_GUI_NIC="" +# Remote GUI Frontend Port +REMOTE_GUI_FE=31800 +# Remote GUI Backend Port +REMOTE_GUI_BE=8080 +# Grafana PATH to reference for Grafana connection +GRAFANA_URL="" + + diff --git a/docker-makefiles/roy-operator1/node_configs.env b/docker-makefiles/roy-operator1/node_configs.env new file mode 100644 index 00000000..de17ec34 --- /dev/null +++ b/docker-makefiles/roy-operator1/node_configs.env @@ -0,0 +1,296 @@ +#==== .env ==== +# Basic deployment settings + +# Container image, deployment type +# -> prod: run AnyLog instance +# -> bash: run shell interface with AnyLog installed and params set +INIT_TYPE=prod + +# AnyLog image +IMAGE=anylogco/anylog-network + +# Github repo to use for configuring / deploying node +DEPLOYMENTS_REPO="/mnt/c/Users/oshad/AnyLog-code/deployment-scripts" +# Branch associated with git repo +DEPLOYMENTS_BRANCH="os-dev" + +# specify the docker socket - to be used when deploying docker monitoring +DOCKER_SOCKET="/var/run/docker.sock" + +#==== basic ==== +# Core node configuration - required for all deployments +#--- General --- +# Configuration based deploymnet node deployment type +# Node Types: +# generic: sandbox with only network configured +# master: "blockchain" emulator +# operator: agent dedicated to storing data +# query: agent dedicated to querying data (enable system_query logical database) +# publisher: agent used to distribute data +# master-operator: a combination of master and operator on a single agent +# master-publisher: a combination of master and publisher on a single agent +NODE_TYPE=operator + +# Name of the AnyLog instance +NODE_NAME=operator1 +# Owner of the AnyLog instance +COMPANY_NAME=AnyLog Co. + +#--- Networking --- +# Port address used by AnyLog's TCP protocol to communicate with other nodes in the network +ANYLOG_SERVER_PORT=32148 +# Port address used by AnyLog's REST protocol +ANYLOG_REST_PORT=32149 +# Port value to be used as an MQTT broker, or some other third-party broker +ANYLOG_BROKER_PORT=32150 + +#--- Database --- +# Physical database type (sqlite, psql) +DB_TYPE=sqlite +# Database IP address +DB_IP=${INET_IP} +# Database port number +DB_PORT=5432 +# Whether to set autocommit data +AUTOCOMMIT=true + +#--- NoSQL --- +# Enable external blob storage +BLOBS_STORAGE=false +# Storage backend type - Supported: mongo, s3, akave, minio +BLOB_STORAGE_TYPE=mongo +# Storage endpoint (MongoDB or object storage endpoint) +BLOB_STORAGE_IP=${INET_IP} +# Storage port (used mainly for MongoDB) +BLOB_STORAGE_PORT=27017 + +#--- System Query --- +# Whether to start system_query logical database +SYSTEM_QUERY=True +# Database type used for system_query logical database +SYSTEM_QUERY_DB=sqlite +# Run system_query using in-memory SQLite. If set to false, will use pre-set database type +MEMORY=True + +#--- Blockchain --- +# TCP connection information for Master Node +LEDGER_CONN=${INET_IP}:32048 +# How often to sync from blockchain +BLOCKCHAIN_SYNC=60 second +# Source of where the blockchain data is coming from (master, optimism) +BLOCKCHAIN_SOURCE=master +# Where the local copy of the blockchain will be stored +BLOCKCHAIN_DESTINATION=file + +#--- Operator --- +# Owner of the cluster +CLUSTER_NAME=anylog-cluster1 +# Logical database name +DEFAULT_DBMS=mydb + +#==== southbound ==== +# Data ingestion protocols (MQTT, OPC-UA, video streaming, monitoring) +#--- Monitoring --- +# Whether to monitor the node or not +NODE_MONITORING=false +# Store monitoring in Operator node(s) +STORE_MONITORING=false +# Accept syslog data from local (Message broker required) +SYSLOG_MONITORING=false +# Docker Monitoring +DOCKER_MONITORING=false + +#--- Video Streaming --- +# Enable or disable video streaming. +# true = stream video, false = do not stream +ENABLE_VIDEO_STREAMING=false +# URL of the video to stream. Can be a YouTube link or other video source. +VIDEO_URL=https://www.youtube.com/watch?v=rnXIjl_Rzy4 +# Port on which the video streaming server will run. +VIDEO_PORT=32800 +# Name of the video stream or service identifier. +VIDEO_NAME=sfstream +# Enable or disable object detections (like YOLO detections) on the video. +# true = enable detections, false = disable +ENABLE_DETECTIONS=false +# Port on which the YOLO model server will run for object detection. +YOLO_MODEL_PORT=50051 + +#--- MQTT --- +# Whether to enable the default MQTT process +ENABLE_MQTT=false + +# IP address of MQTT broker +MQTT_BROKER=172.104.228.251 +# Port associated with MQTT broker +MQTT_PORT=1883 +# Whether to enable MQTT logging process +MQTT_LOG=false + +# Topic to get data for +MSG_TOPIC=anylog-demo +# Logical database name +MSG_DBMS=mydb +# Table where to store data +MSG_TABLE=bring [table] +# Timestamp column name +MSG_TIMESTAMP_COLUMN=bring [timestamp] +# Value column name +MSG_VALUE_COLUMN=bring [value] +# Column value type +MSG_VALUE_COLUMN_TYPE=float + +#--- OPC-UA --- +# Whether or not to enable to OPC-UA service +ENABLE_OPCUA=false +# OPC-UA URL address (ex. opcua.tcp:;//127.0.0.1:4840) +OPCUA_URL="" +# Node information the root is located in (ex. ns=2;s=DataSet) +OPCUA_NODE="" +# How often to pull data from OPC-UA +OPCUA_FREQUENCY=5 + +#==== advanced ==== +# Fine-tuning for experienced users - safe to leave as defaults +#--- Directories --- +# AnyLog Root Path +ANYLOG_PATH=/app +# !local_scripts: ${ANYLOG_HOME}/deployment-scripts/scripts +LOCAL_SCRIPTS=/app/deployment-scripts +# !test_dir: ${ANYLOG_HOME}/deployment-scripts/test +TEST_DIR=/app/deployment-scripts/test + +#--- General --- +# hide blockchain policy - supports all node types except cluster / operator +IS_HIDDEN=false +# Whether to automatically run a local (or personalized) script at the end of the process +DEPLOY_LOCAL_SCRIPT=false +# Coordinates of the machine - used by Grafana to map the network +LOCATION="" +# Country where machine is located +COUNTRY="" +# State where machine is located +STATE="" +# City where machine is located +CITY="" + +#--- Networking --- +# Declare Policy name +CONFIG_NAME="" +# Whether to enable DNS resolution +ENABLE_DNS=false +# NIC type used to resolve the internal IP address (ex. eth0, lo) +NIC_TYPE="" +# Overlay IP address - if set, will replace local IP address when connecting to network +OVERLAY_IP="" +# External DNS resolution +EXTERNAL_DNS="" +# local DNS domain to append to hostname (ex. example.com) +DNS_DOMAIN="" +# Full path for machine's (local) DNS +DNS="" + +# Bind settings - a true value binds to a specific IP and Port (a false value binds to all IPs) +TCP_BIND=false +REST_BIND=false +BROKER_BIND=false + +# Number of concurrent threads per connection protocol +TCP_THREADS=6 +REST_THREADS=6 +BROKER_THREADS=6 + +# Timeout in seconds for REST requests (0 = continuous) +REST_TIMEOUT=30 + +#--- NoSQL --- +# Reuse identical blobs using file hash +BLOBS_REUSE=true +# Compress blobs before storage +BLOBS_COMPRESS=true + +# Folder for storing blobs locally +# Leave empty to use default application path +BLOBS_FOLDER="" + +#--- Object Store --- +# Logical bucket grouping +BUCKET_GROUP="" +# Bucket identifier +BUCKET_ID="" +# Cloud region +BUCKET_REGION="" + +#--- Operator --- +# Operator ID - autogenerated INT if not specified +# This is used to help keep a record of where data was originally stored +MEMBER="" +# Whether of not to enable HA against the cluster +ENABLE_HA=false +# How many days back to sync between nodes +START_DATE=30 +# How many threads to use in the operator process +OPERATOR_THREADS=3 + +#--- Sub processes --- +# Compress JSON and SQL file(s) backup +COMPRESS_FILE=true +# When data comes in write to database immediately, as opposed to waiting for a full buffer +WRITE_IMMEDIATE=true +# If buffer is not full, how long to wait until pushing data through +THRESHOLD_TIME=1 seconds +# Buffer size to reach, at which point data is pushed through +THRESHOLD_VOLUME=10KB +# Number of parallel queries +QUERY_POOL=6 + +#--- DEBUG --- +ENABLE_TRACEBACK=false +TRACE_LEVEL=0 + +#==== secrets ==== +# Sensitive credentials - do not commit to version control +# will be updated to use sercets logic for both docker and k8s + +#--- General --- +# AnyLog License Key +LICENSE_KEY=5528032c1ee582225942bae2631c9adc97213543c9caec371e55c70c5a7c4ed2303cef87ab33e6499a42daf52cfb41a09b7f558334acf08c42d72f122f1cba88164ebeba48f176e2b261b7b8b34d0d404cd7397df7b80b16812706a01f632c6dc6ee27d1b69f39884b68c938242088cc60852103651c092f82e7ba52f4f41430{"company":"Roy","expiration":"2036-01-30","type":"beta"} + +#--- Database --- +# Username for SQL database connection +DB_USER=demo +# Password correlated to database user +DB_PASSWD=passwd + +#--- NoSQL --- +# Optional DB authentication +BLOB_STORAGE_USER=admin +BLOB_STORAGE_PASSWORD=passwd + +#--- MQTT --- +# User associated with MQTT broker +MQTT_USER=anyloguser +# Password associated with MQTT user +MQTT_PASSWD=mqtt4AnyLog! + +#--- Object Store --- +# Access credentials +BUCKET_ACCESS_KEY="" +BUCKET_SECRET_KEY="" + +#==== remote-gui ==== +# Companion UI application - deployed alongside the AnyLog node +# Whether to enable Remote-GUI in deployment +ENABLE_REMOTE_GUI=false +# Remote-GUI version Tag +REMOTE_GUI_TAG=beta2 +# NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 +REMOTE_GUI_NIC="" +# Remote GUI Frontend Port +REMOTE_GUI_FE=31800 +# Remote GUI Backend Port +REMOTE_GUI_BE=8080 +# Grafana PATH to reference for Grafana connection +GRAFANA_URL="" + + diff --git a/openbao/docker-compose.yml b/openbao/docker-compose.yml new file mode 100644 index 00000000..5b195380 --- /dev/null +++ b/openbao/docker-compose.yml @@ -0,0 +1,24 @@ +services: + anylog-node: + image: anylogco/anylog-network:1.3.2412-beta8 + restart: always + container_name: anylog-node + stdin_open: true + tty: true + network_mode: host + environment: + - INIT_TYPE=prod + - OPENBAO_VAULT_IP=http://0.0.0.0:8200 + - OPENBAO_VAULT_TOKEN=s.57hG4fLMZzRbbPF7mF5fA2Md + - SECTION_NAME=mybao + volumes: + - anylog-node-anylog:/app/AnyLog-Network/anylog + - anylog-node-blockchain:/app/AnyLog-Network/blockchain + - anylog-node-data:/app/AnyLog-Network/data + - anylog-node-local-scripts:/app/deployment-scripts + +volumes: + anylog-node-anylog: + anylog-node-blockchain: + anylog-node-data: + anylog-node-local-scripts: \ No newline at end of file diff --git a/openbao/get4bao.sh b/openbao/get4bao.sh new file mode 100644 index 00000000..9a2629b8 --- /dev/null +++ b/openbao/get4bao.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# 0.0.0.0:8200 s.Pb8yAsBHAKyva0DHmQA4Ib12 +python3 get4bao.py ${VAULT_URL} ${VAULT_TOKEN} + +source $ANYLOG_PATH/openbao/configs.env \ No newline at end of file diff --git a/openbao/openbao.env b/openbao/openbao.env new file mode 100644 index 00000000..9bd4ed5f --- /dev/null +++ b/openbao/openbao.env @@ -0,0 +1,9 @@ +# test configs +SERVICE_PORT=32548 +TCP_BIND=true +TCP_THREADS=3 + +REST_PORT=32549 +REST_TIMEOUT=30 +REST_BIND=false +REST_THREADS=3 \ No newline at end of file diff --git a/openbao/openbao_cmds.py b/openbao/openbao_cmds.py new file mode 100644 index 00000000..3b246854 --- /dev/null +++ b/openbao/openbao_cmds.py @@ -0,0 +1,270 @@ +# URL: https://openbao.org/api-docs/ +import argparse +import ast +import json +import os +import requests + +ERRORS_GENERIC = { + 1: "Informational", + 2: "Successful", + 3: "Redirection", + 4: "Client Error", + 5: "Server Error", + 7: "Developer Error" +} + +NETWORK_ERRORS = { + 100: "Continue", + 101: "Switching Protocols", + 200: "OK", + 201: "Created", + 202: "Accepted", + 203: "Non-Authoritative Information", + 204: "No Content", + 205: "Reset Content", + 206: "Partial Content", + 300: "Multiple Choices", + 301: "Moved Permanently", + 302: "Found", + 303: "See Other", + 304: "Not Modified", + 305: "Use Proxy", + 307: "Temporary Redirect", + 400: "Bad Request", + 401: "Unauthorized", + 402: "Payment Required", + 403: "Forbidden", + 404: "Not Found", + 405: "Method Not Allowed", + 406: "Not Acceptable", + 407: "Proxy Authentication Required", + 408: "Request Timeout", + 409: "Conflict", + 410: "Gone", + 411: "Length Required", + 412: "Precondition Failed", + 413: "Payload Too Large", + 414: "URI Too Long", + 415: "Unsupported Media Type", + 416: "Range Not Satisfiable", + 417: "Expectation Failed", + 418: "I'm a teapot", + 426: "Upgrade Required", + 500: "Internal Server Error", + 501: "Not Implemented", + 502: "Bad Gateway", + 503: "Service Unavailable", + 504: "Gateway Time-out", + 505: "HTTP Version Not Supported", + 102: "Processing", + 207: "Multi-Status", + 226: "IM Used", + 308: "Permanent Redirect", + 422: "Unprocessable Entity", + 423: "Locked", + 424: "Failed Dependency", + 428: "Precondition Required", + 429: "Too Many Requests", + 431: "Request Header Fields Too Large", + 451: "Unavailable For Legal Reasons", + 506: "Variant Also Negotiates", + 507: "Insufficient Storage", + 511: "Network Authentication Required" +} + +DEFAULT_CONFIG_FILE = os.path.join(os.path.expandvars(os.path.expanduser(__file__.split("openbao_cmds.py")[0])), 'openbao.env') + + +class OpenBao: + def __init__(self, conn:str, token:str): + self.conn = conn if conn is not None else '0.0.0.0:8200' + self.token = token + if self.token is None: + raise ValueError("Missing token value, cannot continue...") + + def __execute_get(self, url:str, headers:dict): + """ + Execute GET request + :args: + url:str - request URl + headers:dict - REST headers + :params: + response:requests.GET - response fromm GET + :return: + response + if response.status_code has a Client Error, return empty dict + """ + try: + response = requests.get(url=url, headers=headers) + except Exception as error: + raise Exception(f"Failed to execute GET against {url} (Error: {error}") + else: + if 400 <= response.status_code < 500: + response = {} + elif not 200 <= response.status_code < 300: + if int(response.status_code) in NETWORK_ERRORS: + raise ConnectionError(f"Failed to execute GET against {url} (Network Error {response.status_code}: {NETWORK_ERRORS[int(response.status_code)]})") + elif int(response.status_code[0]) in ERRORS_GENERIC: + raise ConnectionError(f"Failed to execute GET against {url} (Network Error {response.status_code}: {ERRORS_GENERIC[int(response.status_code[0])]}") + + return response + + + def get_openbao_status(self): + """ + health check for OpenBao + :anylog-cmd: + get openbao status where name=mybao + :args: + openbao_name:str - OpenBao connection name + :params: + headers:dict - REST headers + url:str - URL to access OpenBao + :return: + dictionary for OpenBao status + :sample output: + { + 'initialized': True, 'sealed': False, 'standby': False, 'performance_standby': False, + 'replication_performance_mode': 'disabled', 'replication_dr_mode': 'disabled', + 'server_time_utc': 1738794866, 'version': '2.1.1', 'cluster_name': 'vault-cluster-8acc91c5', + 'cluster_id': 'bd9670ca-b594-3c6b-6431-2d81afa9bdd0' + } + """ + headers = { + "X-Vault-Token": self.token, + "Content-Type": "application/json" + } + + url = f"http://{self.conn}/v1/sys/health" + response = self.__execute_get(url=url, headers=headers) + try: + return response.json() + except Exception as error: + raise Exception(f"Failed to extract results for {self.conn} (Error: {error})") + + + def get_openbao_value(self, section:str, key:str=None): + """ + Get secret value(s) within OpenBao + :anylog-cmd: + get openbao status where name=mybao and section=XXX [and key=XXX] + :args: + openbao_name:str - OpenBao connection name + section:str - subsection within secrets data key/value pairs will be stored + key:str - specific key to get value for + :params: + headers:dict - REST headers + url:str - URL to access OpenBao + :return: + if key - return specific value + if no key - return full dict of params + :note: + if section or key is invalid then returns error 404 - error should be "failed to get valid data" + """ + headers = { + "X-Vault-Token": self.token, + } + + url = f"http://{self.conn}/v1/secret/data/{section}" + response = self.__execute_get(url=url, headers=headers) + if response == {}: + return response + + try: + output = response.json() + except Exception as error: + raise Exception(f"Failed to extract key/values for {url} (Error: {error})") + + content = output['data']['data'][section] + if key: + if key in content: + return content[key] + else: + raise ValueError(f'Failed to find value for {key}') + else: + return content + + + def publish_data(self, section:str, payload:dict): + """ + Publish value to OpenBao + :args: + :args: + openbao_name:str - OpenBao connection name + section:str - subsection within secrets data key/value pairs will be stored + payload:dict - key/value pairs + :params: + headers:dict - REST headers + url:str - URL to access OpenBao + """ + headers = { + "X-Vault-Token": self.token, + "Content-Type": "application/json" + } + + orig_payloads = self.get_openbao_value(section=section) + updated_payloads = {**orig_payloads, **payload} + publish_payloads = {'data': { + section: updated_payloads + }} + + url = f"http://{self.conn}/v1/secret/data/{section}" + + try: + response = requests.post(url=url, headers=headers, json=publish_payloads) + except Exception as error: + raise Exception(f"Failed to POST data against {url} (Error: {error})") + else: + if not 200 <= response.status_code < 300: + if int(response.status_code) in NETWORK_ERRORS: + raise ConnectionError(f"Failed to execute POST against {url} (Network Error {response.status_code}: {NETWORK_ERRORS[int(response.status_code)]})") + elif int(response.status_code[0]) in ERRORS_GENERIC: + raise ConnectionError(f"Failed to execute POST against {url} (Network Error {response.status_code}: {ERRORS_GENERIC[int(response.status_code[0])]}") + + +def main(): + parse = argparse.ArgumentParser() + parse.add_argument('--config-file', type=str,default=DEFAULT_CONFIG_FILE, help='OpenBao .env config files') + parse.add_argument('--conn', type=str, default='0.0.0.0:8200', help='URL connection') + parse.add_argument('--token', type=str, default=None, help='OpenBao token') + parse.add_argument('--section', type=str, default='mybao', help='Section to secrets data location') + args = parse.parse_args() + + openbao = OpenBao(conn=args.conn, token=args.token) + args.config_file = os.path.expandvars(os.path.expanduser(args.config_file)) + if not os.path.isfile(args.config_file): + raise FileNotFoundError(f"Failed to locate config file {args.config_file}") + + # check status + print(openbao.get_openbao_status()) + configs = {} + with open(args.config_file, 'r') as f: + for line in f.read().split("\n"): + if not (line.startswith("#") or not line): + key, value = line.split("=") + try: + configs[key.strip()] = ast.literal_eval(value.strip()) + except: + configs[key.strip()] = value.strip() + + openbao.publish_data(section=args.section, payload=configs) + print(openbao.get_openbao_value(section=args.section)) + + +if __name__ == '__main__': + main() + # # 127.0.0.1:8200 --check-health + # openbao = OpenBao() + # openbao.connect_openbao(name='mybao', conn='0.0.0.0:8200', token='s.57hG4fLMZzRbbPF7mF5fA2Md') + # + # # check status + # print(openbao.get_openbao_status(openbao_name='mybao')) + # + # # create section and set payload value(s) + # openbao.publish_data(openba_name='mybao', section='anylog-data', payload={'param1': True, 'param2': 3, 'param3': 'abc'}) + # print(openbao.get_openbao_value(openbao_name='mybao', section='anylog-data')) + # + # # add another value and update value to payload + # openbao.publish_data(openba_name='mybao', section='anylog-data', payload={'param1': False, 'param4': ''}) + # print(openbao.get_openbao_value(openbao_name='mybao', section='anylog-data')) \ No newline at end of file diff --git a/openbao/run_openbao.sh b/openbao/run_openbao.sh new file mode 100644 index 00000000..9caa0d63 --- /dev/null +++ b/openbao/run_openbao.sh @@ -0,0 +1,7 @@ +docker run -d \ + --name openbao \ + -p 8080:8080 \ + -p 8200:8200 \ + -v openbao-config:/config \ + -v openbao-data:/data \ +--rm openbao/openbao:latest \ No newline at end of file diff --git a/row_count.sh b/row_count.sh new file mode 100644 index 00000000..7234c067 --- /dev/null +++ b/row_count.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +NODE_CONN=${1:-127.0.0.1:32349} +DB_NAME=${2:-mybd} +TIMESTAMP=${3:-insert_timestamp} + +# check status - fails do not continue +curl -X GET "http://${NODE_CONN}" -w "\n" + +TABLES=$(curl -s -X GET "http://${NODE_CONN}" \ + -H "command: blockchain get table where dbms=${DB_NAME} bring [*][name] separator=\n" \ + -H "User-Agent: AnyLog/1.23" +) + +echo "Current Timestamp (UTC): $(date -u +"%Y-%m-%d %H:%M:%S")" + +for TABLE in ${TABLES}; do + echo "${TABLE}" + + for WHERE_CONDITION in \ + "${TIMESTAMP} >= NOW() - 1 hour" \ + "period(hour, 1, now(), ${TIMESTAMP})" + do + echo "${TABLE} - ${WHERE_CONDITION}" + + curl -s -X GET "http://${NODE_CONN}" \ + -H "command: sql ${DB_NAME} format=table and timezone=utc and extend=(+node_name) \ +SELECT MIN(${TIMESTAMP}) as min_ts, MAX(${TIMESTAMP}), COUNT(*) FROM ${TABLE} WHERE ${WHERE_CONDITION}" \ + -H "User-Agent: AnyLog/1.23" \ + -H "destination: network" \ + -w "\n" + done +done \ No newline at end of file From 5a16c918c96d4a101690d2dc5a3a718071921e63 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 30 Apr 2026 21:17:50 +0000 Subject: [PATCH 04/46] chore: update version for os-dev [skip ci] --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index e80dbf16..0033a95f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = docker-compose -version = 1.4.2604 (pre-develop - 34235b [2026-04-30 00:53:27]) +version = 1.4.2604 (os-dev - d399b0 [2026-04-30 21:17:50]) author = AnyLog Co. contact = info@anylog.co description = docker-compose configurations for AnyLog deployment From 95631efbeade763090691a478f1c6ff48701c66e Mon Sep 17 00:00:00 2001 From: Ori Shadmon Date: Mon, 4 May 2026 13:01:42 -0700 Subject: [PATCH 05/46] env params (ori) Signed-off-by: Ori Shadmon --- deploy.sh | 2 +- docker-makefiles/build_docker_compose.sh | 2 +- docker-makefiles/prep_configs.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy.sh b/deploy.sh index f035d978..e54d0b8e 100644 --- a/deploy.sh +++ b/deploy.sh @@ -2,7 +2,7 @@ # deploy.sh — AnyLog node lifecycle manager # Usage: bash deploy.sh [OPTIONS] # Run: bash deploy.sh help -#set -euo pipefail +set -euo pipefail # ────────────────────────────────────────────── # Defaults (override via environment or flags) diff --git a/docker-makefiles/build_docker_compose.sh b/docker-makefiles/build_docker_compose.sh index 939739bc..757e3c70 100644 --- a/docker-makefiles/build_docker_compose.sh +++ b/docker-makefiles/build_docker_compose.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -euo pipefail +#set -euo pipefail # -------- Helpers -------- die() { diff --git a/docker-makefiles/prep_configs.sh b/docker-makefiles/prep_configs.sh index b13fba3f..30d88654 100644 --- a/docker-makefiles/prep_configs.sh +++ b/docker-makefiles/prep_configs.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -euo pipefail +#set -euo pipefail # -------- Helpers -------- die() { From 987bb3cd3f73ea3c09d16f2f1ae228c2329aaca4 Mon Sep 17 00:00:00 2001 From: Ori Shadmon Date: Mon, 4 May 2026 13:16:20 -0700 Subject: [PATCH 06/46] missing clean compose repo Signed-off-by: Ori Shadmon --- deploy.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/deploy.sh b/deploy.sh index e54d0b8e..487c1756 100644 --- a/deploy.sh +++ b/deploy.sh @@ -92,6 +92,14 @@ _check_configs() { DOCKER_COMPOSE_FILE="docker-makefiles/docker-compose-files/${ANYLOG_TYPE}-docker-compose.yaml" +# remove DOCKER_COMPOSE_FILE if exists +_cmd_clean_compose() { + if [[ "$IS_MANUAL" == "false" ]] && [[ -f ${DOCKER_COMPOSE_FILE} ]]; then + rm -rf "${DOCKER_COMPOSE_FILE}" + fi +} + + # ────────────────────────────────────────────── # Parse flags (--is-manual, --type, --tag, etc.) # ────────────────────────────────────────────── @@ -215,6 +223,7 @@ cmd_down() { fi } + cmd_clean() { _check_configs _load_configs @@ -231,6 +240,7 @@ cmd_clean() { echo "Stopping + removing volumes: ${ANYLOG_TYPE}" ${DOCKER_COMPOSE_CMD} -f "${DOCKER_COMPOSE_FILE}" down -v fi + _cmd_clean_compose } cmd_clean_all() { @@ -250,6 +260,7 @@ cmd_clean_all() { echo "Stopping + removing volumes + image: ${ANYLOG_TYPE}" ${DOCKER_COMPOSE_CMD} -f "${DOCKER_COMPOSE_FILE}" down -v --rmi all fi + _cmd_clean_compose } cmd_logs() { From 0b0804639ef98f8b89385fd30961f5721bee1951 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 4 May 2026 20:21:48 +0000 Subject: [PATCH 07/46] chore: update version for os-dev [skip ci] --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 0033a95f..1ee050da 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = docker-compose -version = 1.4.2604 (os-dev - d399b0 [2026-04-30 21:17:50]) +version = 1.4.2604 (os-dev - 987bb3 [2026-05-04 20:21:48]) author = AnyLog Co. contact = info@anylog.co description = docker-compose configurations for AnyLog deployment From 67b054ccb521aa0e4aab000feb974abc974da15c Mon Sep 17 00:00:00 2001 From: Ori Shadmon Date: Tue, 5 May 2026 11:26:59 -0700 Subject: [PATCH 08/46] ori changes Signed-off-by: Ori Shadmon --- deploy.sh | 2 +- .../node_configs.env | 18 +++++++++++------- support/Makefile | 2 +- support/docker_compose_builder.sh | 2 +- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/deploy.sh b/deploy.sh index 487c1756..6c6607da 100644 --- a/deploy.sh +++ b/deploy.sh @@ -2,7 +2,7 @@ # deploy.sh — AnyLog node lifecycle manager # Usage: bash deploy.sh [OPTIONS] # Run: bash deploy.sh help -set -euo pipefail +#set -euo pipefail # ────────────────────────────────────────────── # Defaults (override via environment or flags) diff --git a/docker-makefiles/anylog-standalone-operator/node_configs.env b/docker-makefiles/anylog-standalone-operator/node_configs.env index 7ccfc082..28b7d778 100644 --- a/docker-makefiles/anylog-standalone-operator/node_configs.env +++ b/docker-makefiles/anylog-standalone-operator/node_configs.env @@ -48,7 +48,7 @@ ANYLOG_BROKER_PORT=32150 #--- Database --- # Physical database type (sqlite, psql) -DB_TYPE=sqlite +DB_TYPE=psql # Database IP address DB_IP=127.0.0.1 # Database port number @@ -94,13 +94,16 @@ DEFAULT_DBMS=new_company # Data ingestion protocols (MQTT, OPC-UA, video streaming, monitoring) #--- Monitoring --- # Whether to monitor the node or not -NODE_MONITORING=false +NODE_MONITORING=true # Store monitoring in Operator node(s) -STORE_MONITORING=false +STORE_MONITORING=true # Accept syslog data from local (Message broker required) -SYSLOG_MONITORING=false +SYSLOG_MONITORING=true # Docker Monitoring -DOCKER_MONITORING=false +DOCKER_MONITORING=true + +# how often data is being pulled from docker +DOCKER_FREQUENCY = 30 #--- Video Streaming --- # Enable or disable video streaming. @@ -240,11 +243,12 @@ COMPRESS_FILE=true # When data comes in write to database immediately, as opposed to waiting for a full buffer WRITE_IMMEDIATE=true # If buffer is not full, how long to wait until pushing data through -THRESHOLD_TIME=60 seconds +THRESHOLD_TIME=10 seconds # Buffer size to reach, at which point data is pushed through -THRESHOLD_VOLUME=10KB +THRESHOLD_VOLUME=1KB # Number of parallel queries QUERY_POOL=6 +# for `run operator` process, specify the number of parallel threads to use #==== secrets ==== # Sensitive credentials - do not commit to version control diff --git a/support/Makefile b/support/Makefile index 51015526..23c71811 100644 --- a/support/Makefile +++ b/support/Makefile @@ -125,7 +125,7 @@ _logs-f: _validate # postgres* -> psql | mongo* -> mongosh | everything else -> bash _attach: _validate @if echo "$(SVC)" | grep -q "^postgres"; then \ - $(CONTAINER_CMD) exec -it $(SVC) psql -U postgres; \ + $(CONTAINER_CMD) exec -it $(SVC) psql -U admin -d postgres; \ elif echo "$(SVC)" | grep -q "^mongo"; then \ $(CONTAINER_CMD) exec -it $(SVC) mongosh; \ else \ diff --git a/support/docker_compose_builder.sh b/support/docker_compose_builder.sh index 73ac2443..b4e08b39 100644 --- a/support/docker_compose_builder.sh +++ b/support/docker_compose_builder.sh @@ -4,7 +4,7 @@ # config_file defaults to configs.yaml # output_file defaults to docker-compose.yml -set -euo pipefail +#set -euo pipefail CONFIG_FILE="${1:-configs.yaml}" OUTPUT_FILE="${2:-docker-compose.yml}" From bead80810f5a45efc6f5abeb978132b90ff90584 Mon Sep 17 00:00:00 2001 From: Ori Shadmon Date: Tue, 5 May 2026 18:02:34 -0700 Subject: [PATCH 09/46] operator helper/worker threads Signed-off-by: Ori Shadmon --- .../anylog-standalone-operator/node_configs.env | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docker-makefiles/anylog-standalone-operator/node_configs.env b/docker-makefiles/anylog-standalone-operator/node_configs.env index 28b7d778..03f3c99f 100644 --- a/docker-makefiles/anylog-standalone-operator/node_configs.env +++ b/docker-makefiles/anylog-standalone-operator/node_configs.env @@ -12,9 +12,9 @@ IMAGE=anylogco/anylog-network # Github repo to use for configuring / deploying node # The path can either be a Github URL and then the code will automatically clone # the repo if needed OR a local path for deployment-scripts and then use that instead of cloning into the repositroy -DEPLOYMENTS_REPO="/mnt/c/Users/oshad/AnyLog-code/deployment-scripts" # Branch associated with git repo -DEPLOYMENTS_BRANCH="pre-develop" +DEPLOYMENTS_REPO="/mnt/c/Users/oshad/AnyLog-code/deployment-scripts" +DEPLOYMENTS_BRANCH="os-dev" # specify the docker socket - to be used when deploying docker monitoring DOCKER_SOCKET="/var/run/docker.sock" @@ -22,7 +22,7 @@ DOCKER_SOCKET="/var/run/docker.sock" #==== basic ==== # Core node configuration - required for all deployments #--- General --- -# Configuration based deploymnet node deployment type +# Configuration based deployment node deployment type # Node Types: # generic: sandbox with only network configured # master: "blockchain" emulator @@ -92,6 +92,7 @@ DEFAULT_DBMS=new_company #==== southbound ==== # Data ingestion protocols (MQTT, OPC-UA, video streaming, monitoring) + #--- Monitoring --- # Whether to monitor the node or not NODE_MONITORING=true @@ -236,7 +237,8 @@ ENABLE_HA=false START_DATE=30 # How many threads to use in the operator process OPERATOR_THREADS=3 - +# Parallel threads to be used in order digest data into AnyLog +OPERATOR_HELPERS=5 #--- Sub processes --- # Compress JSON and SQL file(s) backup COMPRESS_FILE=true @@ -256,7 +258,7 @@ QUERY_POOL=6 #--- General --- # AnyLog License Key -LICENSE_KEY="" +LICENSE_KEY=cd554412f2a4ff540e3426a4f1fcb6d50ab8d9e1130e42957b0338ea45f4303fd160a551c5200b0321101abdc9eb44169e1b1c9d5a9492b16cc870619bb5199ae806a1dfadda93e00f02d30f3a5dabc0a6a7bb8f869e37fa565962eba55cb335397a66903d8fb3014ae1e4eea46c104cad8c787a0d5ae45efbf8728cf04cb805{"company":"Guest","expiration":"2026-08-01","type":"beta"} #--- Database --- # Username for SQL database connection From e2d7bbc28e1d3e8b0c9455d083f26ebd938d39e4 Mon Sep 17 00:00:00 2001 From: Ori Shadmon Date: Thu, 7 May 2026 13:30:59 -0700 Subject: [PATCH 10/46] hidden query + chnage to build process (to test) Signed-off-by: Ori Shadmon --- .../anylog-query-hidden/node_configs.env | 295 ++++++++++++++++++ docker-makefiles/build_docker_compose.sh | 71 ++--- 2 files changed, 324 insertions(+), 42 deletions(-) create mode 100644 docker-makefiles/anylog-query-hidden/node_configs.env diff --git a/docker-makefiles/anylog-query-hidden/node_configs.env b/docker-makefiles/anylog-query-hidden/node_configs.env new file mode 100644 index 00000000..2ab28f85 --- /dev/null +++ b/docker-makefiles/anylog-query-hidden/node_configs.env @@ -0,0 +1,295 @@ +#==== .env ==== +# Basic deployment settings + +# Container image, deployment type +# -> prod: run AnyLog instance +# -> bash: run shell interface with AnyLog installed and params set +INIT_TYPE=prod + +# AnyLog image +IMAGE=anylogco/anylog-network + +# Github repo to use for configuring / deploying node +# The path can either be a Github URL and then the code will automatically clone +# the repo if needed OR a local path for deployment-scripts and then use that instead of cloning into the repositroy +DEPLOYMENTS_REPO=https://github.com/AnyLog-co/deployment-scripts +# Branch associated with git repo +DEPLOYMENTS_BRANCH="pre-develop" + +# specify the docker socket - to be used when deploying docker monitoring +DOCKER_SOCKET="/var/run/docker.sock" + + +#==== basic ==== +# Core node configuration - required for all deployments +#--- General --- +# Configuration based deploymnet node deployment type +# Node Types: +# generic: sandbox with only network configured +# master: "blockchain" emulator +# operator: agent dedicated to storing data +# query: agent dedicated to querying data (enable system_query logical database) +# publisher: agent used to distribute data +# master-operator: a combination of master and operator on a single agent +# master-publisher: a combination of master and publisher on a single agent +NODE_TYPE=query + +# Name of the AnyLog instance +NODE_NAME=anylog-query +# Owner of the AnyLog instance +COMPANY_NAME="AnyLog Co." + +#--- Networking --- +# Port address used by AnyLog's TCP protocol to communicate with other nodes in the network +ANYLOG_SERVER_PORT=32348 +# Port address used by AnyLog's REST protocol +ANYLOG_REST_PORT=32349 +# Port value to be used as an MQTT broker, or some other third-party broker +ANYLOG_BROKER_PORT="" + +#--- Database --- +# Physical database type (sqlite, psql) +DB_TYPE=sqlite +# Database IP address +DB_IP=127.0.0.1 +# Database port number +DB_PORT=5432 +# Whether to set autocommit data +AUTOCOMMIT=true + +#--- NoSQL --- +# Enable external blob storage +BLOBS_STORAGE=false +# Storage backend type - Supported: mongo, s3, akave, minio +BLOB_STORAGE_TYPE=mongo +# Storage endpoint (MongoDB or object storage endpoint) +BLOB_STORAGE_IP=127.0.0.1 +# Storage port (used mainly for MongoDB) +BLOB_STORAGE_PORT=27017 + +#--- System Query --- +# Whether to start system_query logical database +SYSTEM_QUERY=true +# Database type used for system_query logical database +SYSTEM_QUERY_DB=sqlite +# Run system_query using in-memory SQLite. If set to false, will use pre-set database type +MEMORY=true + +#--- Blockchain --- +# TCP connection information for Master Node +LEDGER_CONN=127.0.0.1:32048 +# How often to sync from blockchain +BLOCKCHAIN_SYNC=60 second +# Source of where the blockchain data is coming from (master, optimism) +BLOCKCHAIN_SOURCE=master +# Where the local copy of the blockchain will be stored +BLOCKCHAIN_DESTINATION=file + +#--- Operator --- +# Owner of the cluster +CLUSTER_NAME=anylog-cluster1 +# Logical database name +DEFAULT_DBMS=mydb + +#==== southbound ==== +# Data ingestion protocols (MQTT, OPC-UA, video streaming, monitoring) +#--- Monitoring --- +# Whether to monitor the node or not +NODE_MONITORING=false +# Store monitoring in Operator node(s) +STORE_MONITORING=false +# Accept syslog data from local (Message broker required) +SYSLOG_MONITORING=false +# Docker Monitoring +DOCKER_MONITORING=false + +#--- Video Streaming --- +# Enable or disable video streaming. +# true = stream video, false = do not stream +ENABLE_VIDEO_STREAMING=false +# URL of the video to stream. Can be a YouTube link or other video source. +VIDEO_URL=https://www.youtube.com/watch?v=rnXIjl_Rzy4 +# Port on which the video streaming server will run. +VIDEO_PORT=32800 +# Name of the video stream or service identifier. +VIDEO_NAME=sfstream +# Enable or disable object detections (like YOLO detections) on the video. +# true = enable detections, false = disable +ENABLE_DETECTIONS=false +# Port on which the YOLO model server will run for object detection. +YOLO_MODEL_PORT=50051 + +#--- MQTT --- +# Whether to enable the default MQTT process +ENABLE_MQTT=false + +# IP address of MQTT broker +MQTT_BROKER=172.104.228.251 +# Port associated with MQTT broker +MQTT_PORT=1883 +# Whether to enable MQTT logging process +MQTT_LOG=false + +# Topic to get data for +MSG_TOPIC=anylog-demo +# Logical database name +MSG_DBMS=mydb +# Table where to store data +MSG_TABLE=bring [table] +# Timestamp column name +MSG_TIMESTAMP_COLUMN=bring [timestamp] +# Value column name +MSG_VALUE_COLUMN=bring [value] +# Column value type +MSG_VALUE_COLUMN_TYPE=float + +#--- OPC-UA --- +# Whether or not to enable to OPC-UA service +ENABLE_OPCUA=false +# OPC-UA URL address (ex. opcua.tcp:;//127.0.0.1:4840) +OPCUA_URL="" +# Node information the root is located in (ex. ns=2;s=DataSet) +OPCUA_NODE="" +# How often to pull data from OPC-UA +OPCUA_FREQUENCY=5 + +#==== advanced ==== +# Fine-tuning for experienced users - safe to leave as defaults +#--- Directories --- +# AnyLog Root Path +ANYLOG_PATH=/app +# !local_scripts: ${ANYLOG_HOME}/deployment-scripts/scripts +LOCAL_SCRIPTS=/app/deployment-scripts +# !test_dir: ${ANYLOG_HOME}/deployment-scripts/test +TEST_DIR=/app/deployment-scripts/test + +#--- General --- +# hide blockchain policy - supports all node types except cluster / operator +IS_HIDDEN=true +# Whether to automatically run a local (or personalized) script at the end of the process +DEPLOY_LOCAL_SCRIPT=false +# Coordinates of the machine - used by Grafana to map the network +LOCATION="" +# Country where machine is located +COUNTRY="" +# State where machine is located +STATE="" +# City where machine is located +CITY="" + +#--- Networking --- +# Declare Policy name +CONFIG_NAME="" +# Whether to enable DNS resolution +ENABLE_DNS=false +# NIC type used to resolve the internal IP address (ex. eth0, lo) +NIC_TYPE="" +# Overlay IP address - if set, will replace local IP address when connecting to network +OVERLAY_IP="" +# External DNS resolution +EXTERNAL_DNS="" +# local DNS domain to append to hostname (ex. example.com) +DNS_DOMAIN="" +# Full path for machine's (local) DNS +DNS="" + +# Bind settings - a true value binds to a specific IP and Port (a false value binds to all IPs) +TCP_BIND=true +REST_BIND=false +BROKER_BIND=false + +# Number of concurrent threads per connection protocol +TCP_THREADS=6 +REST_THREADS=6 +BROKER_THREADS=6 + +# Timeout in seconds for REST requests (0 = continuous) +REST_TIMEOUT=30 + +#--- NoSQL --- +# Reuse identical blobs using file hash +BLOBS_REUSE=true +# Compress blobs before storage +BLOBS_COMPRESS=true + +# Folder for storing blobs locally +# Leave empty to use default application path +BLOBS_FOLDER="" + +#--- Object Store --- +# Logical bucket grouping +BUCKET_GROUP="" +# Bucket identifier +BUCKET_ID="" +# Cloud region +BUCKET_REGION="" + +#--- Operator --- +# Operator ID - autogenerated INT if not specified +# This is used to help keep a record of where data was originally stored +MEMBER="" +# Whether of not to enable HA against the cluster +ENABLE_HA=false +# How many days back to sync between nodes +START_DATE=30 +# How many threads to use in the operator process +OPERATOR_THREADS=3 + +#--- Sub processes --- +# Compress JSON and SQL file(s) backup +COMPRESS_FILE=true +# When data comes in write to database immediately, as opposed to waiting for a full buffer +WRITE_IMMEDIATE=true +# If buffer is not full, how long to wait until pushing data through +THRESHOLD_TIME=60 seconds +# Buffer size to reach, at which point data is pushed through +THRESHOLD_VOLUME=10KB +# Number of parallel queries +QUERY_POOL=6 + +#==== secrets ==== +# Sensitive credentials - do not commit to version control +# will be updated to use sercets logic for both docker and k8s + +#--- General --- +# AnyLog License Key +LICENSE_KEY="" + +#--- Database --- +# Username for SQL database connection +DB_USER=admin +# Password correlated to database user +DB_PASSWD=passwd + +#--- NoSQL --- +# Optional DB authentication +BLOB_STORAGE_USER=admin +BLOB_STORAGE_PASSWORD=passwd + +#--- MQTT --- +# User associated with MQTT broker +MQTT_USER=anyloguser +# Password associated with MQTT user +MQTT_PASSWD=mqtt4AnyLog! + +#--- Object Store --- +# Access credentials +BUCKET_ACCESS_KEY="" +BUCKET_SECRET_KEY="" + +#==== remote-gui ==== +# Companion UI application - deployed alongside the AnyLog node +# Whether to enable Remote-GUI in deployment +ENABLE_REMOTE_GUI=true +# Remote-GUI version Tag +REMOTE_GUI_TAG=beta2 +# NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 +REMOTE_GUI_NIC="" +# Remote GUI Frontend Port +REMOTE_GUI_FE=31800 +# Remote GUI Backend Port +REMOTE_GUI_BE=8080 +# Grafana PATH to reference for Grafana connection +GRAFANA_URL="" + + diff --git a/docker-makefiles/build_docker_compose.sh b/docker-makefiles/build_docker_compose.sh index 757e3c70..3ba1cff7 100644 --- a/docker-makefiles/build_docker_compose.sh +++ b/docker-makefiles/build_docker_compose.sh @@ -23,9 +23,6 @@ if [[ "${DEPLOYMENT_TYPE}" == "k8s" ]]; then die "k8s deployment not yet supported" fi -# -------- Run config update first -------- -# bash "$(dirname "$0")/update_configs.sh" "${NODE_CONFIGS}" - # -------- Locate Config Files -------- MULTI_FILE=false if [[ -f "docker-makefiles/${NODE_CONFIGS}/.env" ]] && \ @@ -53,6 +50,11 @@ export DOCKER_SOCKET=$(grep -m1 '^DOCKER_SOCKET=' "$BASE_ENV" | cut -d= -f2- | t #export DOCKER_GID=$(stat -c '%g' ${DOCKER_SOCKET}) export DEPLOYMENTS_REPO=$(grep -m1 '^DEPLOYMENTS_REPO=' "$BASE_ENV" | cut -d= -f2- | tr -d '"\r') +# -------- LICENSE_KEY: prefer env var (set by Makefile), fall back to config file -------- +if [[ -z "${LICENSE_KEY:-}" ]]; then + export LICENSE_KEY=$(grep -m1 '^LICENSE_KEY=' "$BASE_ENV" | cut -d= -f2- | tr -d '"\r') +fi + # -------- Select Template -------- COMPOSE_FILE="docker-makefiles/docker-compose-template.yaml" TEMPLATE_COMPOSE_FILE="docker-makefiles/docker-compose-template-base.yaml" @@ -63,7 +65,7 @@ if [[ "$(uname -s)" != "Linux" ]]; then fi [[ -f "$TEMPLATE_COMPOSE_FILE" ]] || die "$TEMPLATE_COMPOSE_FILE not found" -cp "$TEMPLATE_COMPOSE_FILE" "$COMPOSE_FILE" +cp "$TEMPLATE_COMPOSE_FILE" "${COMPOSE_FILE}" # -------- Inject env_file -------- if [[ "$MULTI_FILE" == "true" ]]; then @@ -71,63 +73,48 @@ if [[ "$MULTI_FILE" == "true" ]]; then -v base="../../docker-makefiles/${NODE_CONFIGS}/base_configs.env" \ -v adv="../../docker-makefiles/${NODE_CONFIGS}/advance_configs.env" \ '/ env_file:/ {print; print " - " env; print " - " base; print " - " adv; next}1' \ - "$COMPOSE_FILE" > temp.yaml && mv temp.yaml "$COMPOSE_FILE" + "${COMPOSE_FILE}" > temp.yaml && mv temp.yaml "${COMPOSE_FILE}" else awk -v cfg="../../docker-makefiles/${NODE_CONFIGS}/node_configs.env" \ '/ env_file:/ {print; print " - " cfg; next}1' \ - "$COMPOSE_FILE" > temp.yaml && mv temp.yaml "$COMPOSE_FILE" + "${COMPOSE_FILE}" > temp.yaml && mv temp.yaml "${COMPOSE_FILE}" fi # -------- Inject Broker Port -------- if [[ "${TEMPLATE_COMPOSE_FILE}" == *"ports"* ]] && [[ -n "${ANYLOG_BROKER_PORT:-}" ]]; then awk -v port="${ANYLOG_BROKER_PORT}:${ANYLOG_BROKER_PORT}" \ '/ ports:/ {print; print " - " port; next}1' \ - "$COMPOSE_FILE" > temp.yaml && mv temp.yaml "$COMPOSE_FILE" + "${COMPOSE_FILE}" > temp.yaml && mv temp.yaml "${COMPOSE_FILE}" fi # -------- Update Volumes -------- if ! ([[ ! -d "${DEPLOYMENTS_REPO}" ]] || [[ -z "$(ls -A "${DEPLOYMENTS_REPO}" 2>/dev/null)" ]]); then - ${SED_INPLACE} "0,/\/app\/deployment-scripts/s#/app/deployment-scripts#\# /app/deployment-scripts#" docker-makefiles/docker-compose-template.yaml - ${SED_INPLACE} "0,/- \${NODE_NAME}-local-scripts/s#- \${NODE_NAME}-local-scripts#\# - ${NODE_NAME}-local-scripts#" docker-makefiles/docker-compose-template.yaml - ${SED_INPLACE} "0,/\${NODE_NAME}-local-scripts/s#\${NODE_NAME}-local-scripts#${DEPLOYMENTS_REPO}#" docker-makefiles/docker-compose-template.yaml - ${SED_INPLACE} "0,/\${NODE_NAME}-local-scripts/s# \${NODE_NAME}-local-scripts#\# - ${NODE_NAME}-local-scripts#" docker-makefiles/docker-compose-template.yaml + ${SED_INPLACE} "0,/\/app\/deployment-scripts/s#/app/deployment-scripts#\# /app/deployment-scripts#" "${COMPOSE_FILE}" + ${SED_INPLACE} "0,/- \${NODE_NAME}-local-scripts/s#- \${NODE_NAME}-local-scripts#\# - ${NODE_NAME}-local-scripts#" "${COMPOSE_FILE}" + ${SED_INPLACE} "0,/\${NODE_NAME}-local-scripts/s#\${NODE_NAME}-local-scripts#${DEPLOYMENTS_REPO}#" "${COMPOSE_FILE}" + ${SED_INPLACE} "0,/\${NODE_NAME}-local-scripts/s# \${NODE_NAME}-local-scripts#\# - ${NODE_NAME}-local-scripts#" "${COMPOSE_FILE}" fi -# if path dne of socket dne then comment out section +# -------- Docker Socket -------- if [[ -z "${DOCKER_SOCKET}" ]] || [[ ! -S "${DOCKER_SOCKET}" ]]; then - # comment out group_add - ${SED_INPLACE} "s/- \${DOCKER_GID}/#- \${MISSING-DOCKER_GID}/g" docker-makefiles/docker-compose-template.yaml - # comment out volume if DNE - ${SED_INPLACE} "0,/- \${DOCKER_SOCKET}/s#- \${DOCKER_SOCKET}#\# - \${MISSING-DOCKER_SOCKET}#" docker-makefiles/docker-compose-template.yaml + ${SED_INPLACE} "s/- \${DOCKER_GID}/#- \${MISSING-DOCKER_GID}/g" "${COMPOSE_FILE}" + ${SED_INPLACE} "0,/- \${DOCKER_SOCKET}/s#- \${DOCKER_SOCKET}#\# - \${MISSING-DOCKER_SOCKET}#" "${COMPOSE_FILE}" else - if stat -c '%g' "${DOCKER_SOCKET}" >/dev/null 2>&1; then - # GNU stat (Linux) - export DOCKER_GID=$(stat -c '%g' "${DOCKER_SOCKET}") + if stat -c '%g' "${DOCKER_SOCKET}" >/dev/null 2>&1; then + export DOCKER_GID=$(stat -c '%g' "${DOCKER_SOCKET}") # GNU stat (Linux) else - # BSD stat (macOS) - export DOCKER_GID=$(stat -f '%g' "${DOCKER_SOCKET}") + export DOCKER_GID=$(stat -f '%g' "${DOCKER_SOCKET}") # BSD stat (macOS) fi fi -awk ' -/^group_add:/ {in_block=1; next} -in_block && /^[[:space:]]*-/ { - if ($0 !~ /^[[:space:]]*#/) { - pass - } -} -in_block && /^[^[:space:]]/ {exit} -END { - sed -} -' docker-makefiles/docker-compose-template.yaml - -if [[ "$(uname)" == "Darwin" ]] ; then - ${SED_INPLACE} 's|pid: "host"|# pid: "host"|g' docker-compose-template.yaml - ${SED_INPLACE} 's|- /proc:/host_proc:ro|# - /proc:/host_proc:ro|g' docker-compose-template.yaml - ${SED_INPLACE} 's|- /:/host:ro|# - /:/host:ro|g' docker-compose-template.yaml - ${SED_INPLACE} 's|- /sys:/host_sys:ro|# - /sys:/host_sys:ro|g' docker-compose-template.yaml +# -------- macOS: comment out Linux-only directives -------- +if [[ "$(uname)" == "Darwin" ]]; then + ${SED_INPLACE} 's|pid: "host"|# pid: "host"|g' "${COMPOSE_FILE}" + ${SED_INPLACE} 's|- /proc:/host_proc:ro|# - /proc:/host_proc:ro|g' "${COMPOSE_FILE}" + ${SED_INPLACE} 's|- /:/host:ro|# - /:/host:ro|g' "${COMPOSE_FILE}" + ${SED_INPLACE} 's|- /sys:/host_sys:ro|# - /sys:/host_sys:ro|g' "${COMPOSE_FILE}" fi + # -------- Remote-GUI -------- if [[ "${ENABLE_REMOTE_GUI}" == "true" ]]; then export REMOTE_GUI_NIC=$(grep -m1 '^REMOTE_GUI_NIC=' "$ENV_FILE" | cut -d= -f2- | tr -d '"\r') @@ -166,7 +153,7 @@ if [[ "${ENABLE_REMOTE_GUI}" == "true" ]]; then }1 END { print " image-vol:"; print " usr-mgm-vol:"; print " report-configs:"; -}' "$COMPOSE_FILE" > temp.yaml && mv temp.yaml "$COMPOSE_FILE" +}' "${COMPOSE_FILE}" > temp.yaml && mv temp.yaml "${COMPOSE_FILE}" # Add remote-gui service awk -v remote_ip="$REMOTE_GUI_IP" \ @@ -198,14 +185,14 @@ END { print " - usr-mgm-vol:/app/CLI/local-cli/backend/usr-mgm/"; print " - report-configs:/app/CLI/local-cli-backend/plugins/reportgenerator/templates"; next -}1' "$COMPOSE_FILE" > temp.yaml && mv temp.yaml "$COMPOSE_FILE" +}1' "${COMPOSE_FILE}" > temp.yaml && mv temp.yaml "${COMPOSE_FILE}" fi # -------- Envsubst & Write Output -------- echo "Generating final docker-compose.yaml..." mkdir -p docker-makefiles/docker-compose-files OUTPUT_FILE="docker-makefiles/docker-compose-files/${NODE_CONFIGS}-docker-compose.yaml" -envsubst < "$COMPOSE_FILE" > "$OUTPUT_FILE" +envsubst < "${COMPOSE_FILE}" > "$OUTPUT_FILE" rm -rf ${COMPOSE_FILE} ${COMPOSE_FILE}.bak echo "Saved: ${OUTPUT_FILE}" From 91d1d6c7a8000f3b34a23753c7c86dee50d38951 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 7 May 2026 20:31:29 +0000 Subject: [PATCH 11/46] chore: update version for os-dev [skip ci] --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 1ee050da..f9047bf8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = docker-compose -version = 1.4.2604 (os-dev - 987bb3 [2026-05-04 20:21:48]) +version = 1.4.2604 (os-dev - e2d7bb [2026-05-07 20:31:29]) author = AnyLog Co. contact = info@anylog.co description = docker-compose configurations for AnyLog deployment From 6a44967caf85e93712beddb5af40e4e7253dd3d3 Mon Sep 17 00:00:00 2001 From: Ori Shadmon Date: Fri, 8 May 2026 13:52:02 -0700 Subject: [PATCH 12/46] support for adding personal scripts into docker container Signed-off-by: Ori Shadmon --- .../node_configs.env | 25 +++++++++++++++-- docker-makefiles/build_docker_compose.sh | 27 +++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/docker-makefiles/anylog-standalone-operator/node_configs.env b/docker-makefiles/anylog-standalone-operator/node_configs.env index 03f3c99f..3170d0dc 100644 --- a/docker-makefiles/anylog-standalone-operator/node_configs.env +++ b/docker-makefiles/anylog-standalone-operator/node_configs.env @@ -19,6 +19,27 @@ DEPLOYMENTS_BRANCH="os-dev" # specify the docker socket - to be used when deploying docker monitoring DOCKER_SOCKET="/var/run/docker.sock" +# Local folders to mount inside the AnyLog container. +# +# Examples: +# No volumes: +# USER_VOLUMES="" +# +# Single volume: +# USER_VOLUMES="/home/user/grpc-scripts" +# +# Multiple volumes (space-separated): +# USER_VOLUMES="/home/user/grpc-scripts /home/user/local-deployments" +# +# Notes: +# - Use absolute paths only +# - Paths are mounted automatically under /app/ +# +# Example: +# /home/user/grpc-scripts -> /app/grpc-scripts +# +USER_VOLUMES="" + #==== basic ==== # Core node configuration - required for all deployments #--- General --- @@ -48,7 +69,7 @@ ANYLOG_BROKER_PORT=32150 #--- Database --- # Physical database type (sqlite, psql) -DB_TYPE=psql +DB_TYPE=sqlite # Database IP address DB_IP=127.0.0.1 # Database port number @@ -197,7 +218,7 @@ DNS_DOMAIN="" DNS="" # Bind settings - a true value binds to a specific IP and Port (a false value binds to all IPs) -TCP_BIND=true +TCP_BIND=false REST_BIND=false BROKER_BIND=false diff --git a/docker-makefiles/build_docker_compose.sh b/docker-makefiles/build_docker_compose.sh index 3ba1cff7..65cb44fe 100644 --- a/docker-makefiles/build_docker_compose.sh +++ b/docker-makefiles/build_docker_compose.sh @@ -49,6 +49,8 @@ export ANYLOG_BROKER_PORT=$(grep -m1 '^ANYLOG_BROKER_PORT=' "$BASE_ENV" | cut -d export DOCKER_SOCKET=$(grep -m1 '^DOCKER_SOCKET=' "$BASE_ENV" | cut -d= -f2- | tr -d '"\r') #export DOCKER_GID=$(stat -c '%g' ${DOCKER_SOCKET}) export DEPLOYMENTS_REPO=$(grep -m1 '^DEPLOYMENTS_REPO=' "$BASE_ENV" | cut -d= -f2- | tr -d '"\r') +export USER_VOLUMES=$(grep -m1 '^USER_VOLUMES=' ${BASE_ENV} | cut -d= -f2- | tr -d '"\r') + # -------- LICENSE_KEY: prefer env var (set by Makefile), fall back to config file -------- if [[ -z "${LICENSE_KEY:-}" ]]; then @@ -95,6 +97,7 @@ if ! ([[ ! -d "${DEPLOYMENTS_REPO}" ]] || [[ -z "$(ls -A "${DEPLOYMENTS_REPO}" 2 ${SED_INPLACE} "0,/\${NODE_NAME}-local-scripts/s# \${NODE_NAME}-local-scripts#\# - ${NODE_NAME}-local-scripts#" "${COMPOSE_FILE}" fi + # -------- Docker Socket -------- if [[ -z "${DOCKER_SOCKET}" ]] || [[ ! -S "${DOCKER_SOCKET}" ]]; then ${SED_INPLACE} "s/- \${DOCKER_GID}/#- \${MISSING-DOCKER_GID}/g" "${COMPOSE_FILE}" @@ -107,6 +110,7 @@ else fi fi + # -------- macOS: comment out Linux-only directives -------- if [[ "$(uname)" == "Darwin" ]]; then ${SED_INPLACE} 's|pid: "host"|# pid: "host"|g' "${COMPOSE_FILE}" @@ -188,6 +192,29 @@ END { }1' "${COMPOSE_FILE}" > temp.yaml && mv temp.yaml "${COMPOSE_FILE}" fi +# -------- USER VOLUMES -------- +if [[ -n "${USER_VOLUMES}" ]]; then + for VOLUME in ${USER_VOLUMES}; do + + if [[ "${VOLUME}" == *"/"* ]]; then + MOUNT_NAME=$(basename "${VOLUME}") + elif [[ "${VOLUME}" == *"\\"* ]]; then + MOUNT_NAME=$(echo "${VOLUME}" | awk -F "\\" '{print $NF}') + else + MOUNT_NAME="${VOLUME}" + fi + + INJECT=" - ${VOLUME:+${VOLUME}:}/app/${MOUNT_NAME}" + + # Insert BEFORE the root-level volumes: + ${SED_INPLACE} "/^volumes:/i\\ +${INJECT} +" "${COMPOSE_FILE}" + + done +fi + + # -------- Envsubst & Write Output -------- echo "Generating final docker-compose.yaml..." mkdir -p docker-makefiles/docker-compose-files From 290a691d0580acaa0db7521f2c14275f5418b4f0 Mon Sep 17 00:00:00 2001 From: Ori Shadmon Date: Fri, 8 May 2026 14:05:51 -0700 Subject: [PATCH 13/46] merge from md-dev Signed-off-by: Ori Shadmon --- .gitignore | 3 +- docker-makefiles/build_docker_compose.sh | 12 ++------ docker-makefiles/prep_configs.sh | 36 +++++++++++++++--------- 3 files changed, 27 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index 786af919..93570a82 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ __pycache__/ docker-makefiles/docker-compose-files support/*/docker-compose.yml support/*/docker-compose.yaml -*venv*/ \ No newline at end of file +*venv*/ +.license* diff --git a/docker-makefiles/build_docker_compose.sh b/docker-makefiles/build_docker_compose.sh index 65cb44fe..b83be99c 100644 --- a/docker-makefiles/build_docker_compose.sh +++ b/docker-makefiles/build_docker_compose.sh @@ -47,7 +47,6 @@ export ANYLOG_SERVER_PORT=$(grep -m1 '^ANYLOG_SERVER_PORT=' "$BASE_ENV" | cut -d export ANYLOG_REST_PORT=$(grep -m1 '^ANYLOG_REST_PORT=' "$BASE_ENV" | cut -d= -f2- | tr -d '"\r') export ANYLOG_BROKER_PORT=$(grep -m1 '^ANYLOG_BROKER_PORT=' "$BASE_ENV" | cut -d= -f2- | tr -d '"\r') export DOCKER_SOCKET=$(grep -m1 '^DOCKER_SOCKET=' "$BASE_ENV" | cut -d= -f2- | tr -d '"\r') -#export DOCKER_GID=$(stat -c '%g' ${DOCKER_SOCKET}) export DEPLOYMENTS_REPO=$(grep -m1 '^DEPLOYMENTS_REPO=' "$BASE_ENV" | cut -d= -f2- | tr -d '"\r') export USER_VOLUMES=$(grep -m1 '^USER_VOLUMES=' ${BASE_ENV} | cut -d= -f2- | tr -d '"\r') @@ -62,7 +61,6 @@ COMPOSE_FILE="docker-makefiles/docker-compose-template.yaml" TEMPLATE_COMPOSE_FILE="docker-makefiles/docker-compose-template-base.yaml" if [[ "$(uname -s)" != "Linux" ]]; then -# TEMPLATE_COMPOSE_FILE="docker-makefiles/docker-compose-template-ports-base.yaml" TEMPLATE_COMPOSE_FILE="docker-makefiles/docker-compose-template-base.yaml" fi @@ -97,7 +95,6 @@ if ! ([[ ! -d "${DEPLOYMENTS_REPO}" ]] || [[ -z "$(ls -A "${DEPLOYMENTS_REPO}" 2 ${SED_INPLACE} "0,/\${NODE_NAME}-local-scripts/s# \${NODE_NAME}-local-scripts#\# - ${NODE_NAME}-local-scripts#" "${COMPOSE_FILE}" fi - # -------- Docker Socket -------- if [[ -z "${DOCKER_SOCKET}" ]] || [[ ! -S "${DOCKER_SOCKET}" ]]; then ${SED_INPLACE} "s/- \${DOCKER_GID}/#- \${MISSING-DOCKER_GID}/g" "${COMPOSE_FILE}" @@ -110,7 +107,6 @@ else fi fi - # -------- macOS: comment out Linux-only directives -------- if [[ "$(uname)" == "Darwin" ]]; then ${SED_INPLACE} 's|pid: "host"|# pid: "host"|g' "${COMPOSE_FILE}" @@ -127,7 +123,7 @@ if [[ "${ENABLE_REMOTE_GUI}" == "true" ]]; then export REMOTE_GUI_TAG=$(grep -m1 '^REMOTE_GUI_TAG=' "$ENV_FILE" | cut -d= -f2- | tr -d '"\r') export GRAFANA_URL=$(grep -m1 '^GRAFANA_URL=' "$ENV_FILE" | cut -d= -f2- | tr -d '"\r') export REMOTE_CONN=$(grep -m1 '^REMOTE_CONN=' "$ENV_FILE" | cut -d= -f2- | tr -d '"\r') - export OVERLAY_IP=$(grpe -m1 '^OVERLAY_IP=' "$ENV_FILE" | cut -d= -f2- | tr -d '"\r') + export OVERLAY_IP=$(grep -m1 '^OVERLAY_IP=' "$ENV_FILE" | cut -d= -f2- | tr -d '"\r') REMOTE_GUI_FE="${REMOTE_GUI_FE:-31800}" REMOTE_GUI_BE="${REMOTE_GUI_BE:-8080}" @@ -143,9 +139,9 @@ if [[ "${ENABLE_REMOTE_GUI}" == "true" ]]; then fi fi - if [[ ! -n "${REMOTE_CONN:-}" ]] && [[ -n "${OVERLAY_IP}" ]] ; then + if [[ ! -n "${REMOTE_CONN:-}" ]] && [[ -n "${OVERLAY_IP}" ]]; then export REMOTE_CONN="${OVERLAY_IP}:${ANYLOG_REST_PORT}" - elif [[ ! -n "${REMOTE_CONN:-}" ]] ; then + elif [[ ! -n "${REMOTE_CONN:-}" ]]; then export REMOTE_CONN="${REMOTE_GUI_IP}:${ANYLOG_REST_PORT}" fi @@ -222,5 +218,3 @@ OUTPUT_FILE="docker-makefiles/docker-compose-files/${NODE_CONFIGS}-docker-compos envsubst < "${COMPOSE_FILE}" > "$OUTPUT_FILE" rm -rf ${COMPOSE_FILE} ${COMPOSE_FILE}.bak echo "Saved: ${OUTPUT_FILE}" - - diff --git a/docker-makefiles/prep_configs.sh b/docker-makefiles/prep_configs.sh index 30d88654..c3c4f30a 100644 --- a/docker-makefiles/prep_configs.sh +++ b/docker-makefiles/prep_configs.sh @@ -67,21 +67,29 @@ fi if [[ ! -f "${BASE_ENV}" ]]; then echo "Failed to locate file: ${BASE_ENV} — skipping LICENSE_KEY update" else - # Extract raw value (everything after LICENSE_KEY=) - CURRENT_LICENSE_KEY=$(sed -n 's/^LICENSE_KEY=//p' "${BASE_ENV}") - - # Strip trailing carriage return and any surrounding quotes - CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY%$'\r'}" - CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY#\"}" ; CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY%\"}" - CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY#\'}" ; CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY%\'}" - - if [[ -z "${CURRENT_LICENSE_KEY}" ]]; then - echo "No LICENSE_KEY found in ${BASE_ENV} — skipping" + # If LICENSE_KEY is set in the environment (passed from Makefile), write it + # into the config file so docker compose env_file picks it up too. + if [[ -n "${LICENSE_KEY:-}" ]]; then + if grep -q '^LICENSE_KEY=' "${BASE_ENV}"; then + sedi "s|^LICENSE_KEY=.*|LICENSE_KEY=${LICENSE_KEY}|" "${BASE_ENV}" + echo "LICENSE_KEY written to ${BASE_ENV} from environment" + else + echo "LICENSE_KEY=${LICENSE_KEY}" >> "${BASE_ENV}" + echo "LICENSE_KEY appended to ${BASE_ENV} from environment" + fi else - # Replace any internal single quotes with double quotes - UPDATED_LICENSE_KEY="${CURRENT_LICENSE_KEY//\'/\"}" - sedi "s|^LICENSE_KEY=.*|LICENSE_KEY=${UPDATED_LICENSE_KEY}|" "${BASE_ENV}" - echo "LICENSE_KEY updated in ${BASE_ENV}" + # Fall back: normalize whatever is already in the file + CURRENT_LICENSE_KEY=$(sed -n 's/^LICENSE_KEY=//p' "${BASE_ENV}") + CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY%$'\r'}" + CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY#\"}" ; CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY%\"}" + CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY#\'}" ; CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY%\'}" + if [[ -z "${CURRENT_LICENSE_KEY}" ]]; then + echo "No LICENSE_KEY in environment or ${BASE_ENV} — skipping" + else + UPDATED_LICENSE_KEY="${CURRENT_LICENSE_KEY//\'/\"}" + sedi "s|^LICENSE_KEY=.*|LICENSE_KEY=${UPDATED_LICENSE_KEY}|" "${BASE_ENV}" + echo "LICENSE_KEY updated in ${BASE_ENV}" + fi fi fi From 3bd2ff9e710681f67faec4567ed13c6cf66b90fb Mon Sep 17 00:00:00 2001 From: Ori Shadmon Date: Fri, 8 May 2026 14:06:41 -0700 Subject: [PATCH 14/46] rm unused docker env files Signed-off-by: Ori Shadmon --- docker-makefiles/master/node_configs.env | 289 ----------------- docker-makefiles/operator1/node_configs.env | 289 ----------------- docker-makefiles/operator2/node_configs.env | 289 ----------------- docker-makefiles/operator3/node_configs.env | 289 ----------------- docker-makefiles/operator4/node_configs.env | 289 ----------------- docker-makefiles/roy-master/node_configs.env | 296 ------------------ .../roy-operator1/node_configs.env | 296 ------------------ 7 files changed, 2037 deletions(-) delete mode 100644 docker-makefiles/master/node_configs.env delete mode 100644 docker-makefiles/operator1/node_configs.env delete mode 100644 docker-makefiles/operator2/node_configs.env delete mode 100644 docker-makefiles/operator3/node_configs.env delete mode 100644 docker-makefiles/operator4/node_configs.env delete mode 100644 docker-makefiles/roy-master/node_configs.env delete mode 100644 docker-makefiles/roy-operator1/node_configs.env diff --git a/docker-makefiles/master/node_configs.env b/docker-makefiles/master/node_configs.env deleted file mode 100644 index 4b232fae..00000000 --- a/docker-makefiles/master/node_configs.env +++ /dev/null @@ -1,289 +0,0 @@ -#==== .env ==== -# Basic deployment settings - -# Container image, deployment type -# -> prod: run AnyLog instance -# -> bash: run shell interface with AnyLog installed and params set -INIT_TYPE=prod - -# AnyLog image -IMAGE=anylogco/anylog-network - -# Github repo to use for configuring / deploying node -DEPLOYMENTS_REPO="https://github.com/AnyLog-co/deployment-scripts" -# Branch associated with git repo -DEPLOYMENTS_BRANCH="pre-develop" - -#==== basic ==== -# Core node configuration - required for all deployments -#--- General --- -# Configuration based deploymnet node deployment type -# Node Types: -# generic: sandbox with only network configured -# master: "blockchain" emulator -# operator: agent dedicated to storing data -# query: agent dedicated to querying data (enable system_query logical database) -# publisher: agent used to distribute data -# master-operator: a combination of master and operator on a single agent -# master-publisher: a combination of master and publisher on a single agent -NODE_TYPE=master - -# Name of the AnyLog instance -NODE_NAME=master -# Owner of the AnyLog instance -COMPANY_NAME=AnyLog Co. - -#--- Networking --- -# Port address used by AnyLog's TCP protocol to communicate with other nodes in the network -ANYLOG_SERVER_PORT=32048 -# Port address used by AnyLog's REST protocol -ANYLOG_REST_PORT=32049 -# Port value to be used as an MQTT broker, or some other third-party broker -ANYLOG_BROKER_PORT="" - -#--- Database --- -# Physical database type (sqlite, psql) -DB_TYPE=sqlite -# Database IP address -DB_IP=127.0.0.1 -# Database port number -DB_PORT=5432 -# Whether to set autocommit data -AUTOCOMMIT=true - -#--- NoSQL --- -# Enable external blob storage -BLOBS_STORAGE=false -# Storage backend type - Supported: mongo, s3, akave, minio -BLOB_STORAGE_TYPE=mongo -# Storage endpoint (MongoDB or object storage endpoint) -BLOB_STORAGE_IP=127.0.0.1 -# Storage port (used mainly for MongoDB) -BLOB_STORAGE_PORT=27017 - -#--- System Query --- -# Whether to start system_query logical database -SYSTEM_QUERY=false -# Database type used for system_query logical database -SYSTEM_QUERY_DB=sqlite -# Run system_query using in-memory SQLite. If set to false, will use pre-set database type -MEMORY=false - -#--- Blockchain --- -# TCP connection information for Master Node -LEDGER_CONN=127.0.0.1:32048 -# How often to sync from blockchain -BLOCKCHAIN_SYNC=60 second -# Source of where the blockchain data is coming from (master, optimism) -BLOCKCHAIN_SOURCE=master -# Where the local copy of the blockchain will be stored -BLOCKCHAIN_DESTINATION=file - -#--- Operator --- -# Owner of the cluster -CLUSTER_NAME=anylog-cluster1 -# Logical database name -DEFAULT_DBMS=mydb - -#==== southbound ==== -# Data ingestion protocols (MQTT, OPC-UA, video streaming, monitoring) -#--- Monitoring --- -# Whether to monitor the node or not -NODE_MONITORING=false -# Store monitoring in Operator node(s) -STORE_MONITORING=false -# Accept syslog data from local (Message broker required) -SYSLOG_MONITORING=false -# Docker Monitoring -DOCKER_MONITORING=false - -#--- Video Streaming --- -# Enable or disable video streaming. -# true = stream video, false = do not stream -ENABLE_VIDEO_STREAMING=false -# URL of the video to stream. Can be a YouTube link or other video source. -VIDEO_URL=https://www.youtube.com/watch?v=rnXIjl_Rzy4 -# Port on which the video streaming server will run. -VIDEO_PORT=32800 -# Name of the video stream or service identifier. -VIDEO_NAME=sfstream -# Enable or disable object detections (like YOLO detections) on the video. -# true = enable detections, false = disable -ENABLE_DETECTIONS=false -# Port on which the YOLO model server will run for object detection. -YOLO_MODEL_PORT=50051 - -#--- MQTT --- -# Whether to enable the default MQTT process -ENABLE_MQTT=false - -# IP address of MQTT broker -MQTT_BROKER=172.104.228.251 -# Port associated with MQTT broker -MQTT_PORT=1883 -# Whether to enable MQTT logging process -MQTT_LOG=false - -# Topic to get data for -MSG_TOPIC=anylog-demo -# Logical database name -MSG_DBMS=mydb -# Table where to store data -MSG_TABLE=bring [table] -# Timestamp column name -MSG_TIMESTAMP_COLUMN=bring [timestamp] -# Value column name -MSG_VALUE_COLUMN=bring [value] -# Column value type -MSG_VALUE_COLUMN_TYPE=float - -#--- OPC-UA --- -# Whether or not to enable to OPC-UA service -ENABLE_OPCUA=false -# OPC-UA URL address (ex. opcua.tcp:;//127.0.0.1:4840) -OPCUA_URL="" -# Node information the root is located in (ex. ns=2;s=DataSet) -OPCUA_NODE="" -# How often to pull data from OPC-UA -OPCUA_FREQUENCY=5 - -#==== advanced ==== -# Fine-tuning for experienced users - safe to leave as defaults -#--- Directories --- -# AnyLog Root Path -ANYLOG_PATH=/app -# !local_scripts: ${ANYLOG_HOME}/deployment-scripts/scripts -LOCAL_SCRIPTS=/app/deployment-scripts -# !test_dir: ${ANYLOG_HOME}/deployment-scripts/test -TEST_DIR=/app/deployment-scripts/test - -#--- General --- -# hide blockchain policy - supports all node types except cluster / operator -IS_HIDDEN=false -# Whether to automatically run a local (or personalized) script at the end of the process -DEPLOY_LOCAL_SCRIPT=false -# Coordinates of the machine - used by Grafana to map the network -LOCATION="" -# Country where machine is located -COUNTRY="" -# State where machine is located -STATE="" -# City where machine is located -CITY="" - -#--- Networking --- -# Declare Policy name -CONFIG_NAME="" -# Whether to enable DNS resolution -ENABLE_DNS=false -# NIC type used to resolve the internal IP address (ex. eth0, lo) -NIC_TYPE="" -# Overlay IP address - if set, will replace local IP address when connecting to network -OVERLAY_IP="192.168.0.140" -# External DNS resolution -EXTERNAL_DNS="" -# local DNS domain to append to hostname (ex. example.com) -DNS_DOMAIN="" -# Full path for machine's (local) DNS -DNS="" - -# Bind settings - a true value binds to a specific IP and Port (a false value binds to all IPs) -TCP_BIND=false -REST_BIND=false -BROKER_BIND=false - -# Number of concurrent threads per connection protocol -TCP_THREADS=6 -REST_THREADS=6 -BROKER_THREADS=6 - -# Timeout in seconds for REST requests (0 = continuous) -REST_TIMEOUT=30 - -#--- NoSQL --- -# Reuse identical blobs using file hash -BLOBS_REUSE=true -# Compress blobs before storage -BLOBS_COMPRESS=true - -# Folder for storing blobs locally -# Leave empty to use default application path -BLOBS_FOLDER="" - -#--- Object Store --- -# Logical bucket grouping -BUCKET_GROUP="" -# Bucket identifier -BUCKET_ID="" -# Cloud region -BUCKET_REGION="" - -#--- Operator --- -# Operator ID - autogenerated INT if not specified -# This is used to help keep a record of where data was originally stored -MEMBER="" -# Whether of not to enable HA against the cluster -ENABLE_HA=false -# How many days back to sync between nodes -START_DATE=30 -# How many threads to use in the operator process -OPERATOR_THREADS=3 - -#--- Sub processes --- -# Compress JSON and SQL file(s) backup -COMPRESS_FILE=true -# When data comes in write to database immediately, as opposed to waiting for a full buffer -WRITE_IMMEDIATE=true -# If buffer is not full, how long to wait until pushing data through -THRESHOLD_TIME=60 seconds -# Buffer size to reach, at which point data is pushed through -THRESHOLD_VOLUME=10KB -# Number of parallel queries -QUERY_POOL=6 - -#==== secrets ==== -# Sensitive credentials - do not commit to version control -# will be updated to use sercets logic for both docker and k8s - -#--- General --- -# AnyLog License Key -LICENSE_KEY=5528032c1ee582225942bae2631c9adc97213543c9caec371e55c70c5a7c4ed2303cef87ab33e6499a42daf52cfb41a09b7f558334acf08c42d72f122f1cba88164ebeba48f176e2b261b7b8b34d0d404cd7397df7b80b16812706a01f632c6dc6ee27d1b69f39884b68c938242088cc60852103651c092f82e7ba52f4f41430{"company":"Roy","expiration":"2036-01-30","type":"beta"} - -#--- Database --- -# Username for SQL database connection -DB_USER=admin -# Password correlated to database user -DB_PASSWD=passwd - -#--- NoSQL --- -# Optional DB authentication -BLOB_STORAGE_USER=admin -BLOB_STORAGE_PASSWORD=passwd - -#--- MQTT --- -# User associated with MQTT broker -MQTT_USER=anyloguser -# Password associated with MQTT user -MQTT_PASSWD=mqtt4AnyLog! - -#--- Object Store --- -# Access credentials -BUCKET_ACCESS_KEY="" -BUCKET_SECRET_KEY="" - -#==== remote-gui ==== -# Companion UI application - deployed alongside the AnyLog node -# Whether to enable Remote-GUI in deployment -ENABLE_REMOTE_GUI=false -# Remote-GUI version Tag -REMOTE_GUI_TAG=beta2 -# NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 -REMOTE_GUI_NIC="" -# Remote GUI Frontend Port -REMOTE_GUI_FE=31800 -# Remote GUI Backend Port -REMOTE_GUI_BE=8080 -# Grafana PATH to reference for Grafana connection -GRAFANA_URL="" - - diff --git a/docker-makefiles/operator1/node_configs.env b/docker-makefiles/operator1/node_configs.env deleted file mode 100644 index 887792bd..00000000 --- a/docker-makefiles/operator1/node_configs.env +++ /dev/null @@ -1,289 +0,0 @@ -#==== .env ==== -# Basic deployment settings - -# Container image, deployment type -# -> prod: run AnyLog instance -# -> bash: run shell interface with AnyLog installed and params set -INIT_TYPE=prod - -# AnyLog image -IMAGE=anylogco/anylog-network - -# Github repo to use for configuring / deploying node -DEPLOYMENTS_REPO="https://github.com/AnyLog-co/deployment-scripts" -# Branch associated with git repo -DEPLOYMENTS_BRANCH="pre-develop" - -#==== basic ==== -# Core node configuration - required for all deployments -#--- General --- -# Configuration based deploymnet node deployment type -# Node Types: -# generic: sandbox with only network configured -# master: "blockchain" emulator -# operator: agent dedicated to storing data -# query: agent dedicated to querying data (enable system_query logical database) -# publisher: agent used to distribute data -# master-operator: a combination of master and operator on a single agent -# master-publisher: a combination of master and publisher on a single agent -NODE_TYPE=operator - -# Name of the AnyLog instance -NODE_NAME=operator1 -# Owner of the AnyLog instance -COMPANY_NAME=AnyLog Co. - -#--- Networking --- -# Port address used by AnyLog's TCP protocol to communicate with other nodes in the network -ANYLOG_SERVER_PORT=32148 -# Port address used by AnyLog's REST protocol -ANYLOG_REST_PORT=32149 -# Port value to be used as an MQTT broker, or some other third-party broker -ANYLOG_BROKER_PORT=32150 - -#--- Database --- -# Physical database type (sqlite, psql) -DB_TYPE=sqlite -# Database IP address -DB_IP=${INET_IP} -# Database port number -DB_PORT=5432 -# Whether to set autocommit data -AUTOCOMMIT=true - -#--- NoSQL --- -# Enable external blob storage -BLOBS_STORAGE=false -# Storage backend type - Supported: mongo, s3, akave, minio -BLOB_STORAGE_TYPE=mongo -# Storage endpoint (MongoDB or object storage endpoint) -BLOB_STORAGE_IP=${INET_IP} -# Storage port (used mainly for MongoDB) -BLOB_STORAGE_PORT=27017 - -#--- System Query --- -# Whether to start system_query logical database -SYSTEM_QUERY=True -# Database type used for system_query logical database -SYSTEM_QUERY_DB=sqlite -# Run system_query using in-memory SQLite. If set to false, will use pre-set database type -MEMORY=True - -#--- Blockchain --- -# TCP connection information for Master Node -LEDGER_CONN=${INET_IP}:32048 -# How often to sync from blockchain -BLOCKCHAIN_SYNC=60 second -# Source of where the blockchain data is coming from (master, optimism) -BLOCKCHAIN_SOURCE=master -# Where the local copy of the blockchain will be stored -BLOCKCHAIN_DESTINATION=file - -#--- Operator --- -# Owner of the cluster -CLUSTER_NAME=anylog-cluster1 -# Logical database name -DEFAULT_DBMS=mydb - -#==== southbound ==== -# Data ingestion protocols (MQTT, OPC-UA, video streaming, monitoring) -#--- Monitoring --- -# Whether to monitor the node or not -NODE_MONITORING=false -# Store monitoring in Operator node(s) -STORE_MONITORING=false -# Accept syslog data from local (Message broker required) -SYSLOG_MONITORING=false -# Docker Monitoring -DOCKER_MONITORING=false - -#--- Video Streaming --- -# Enable or disable video streaming. -# true = stream video, false = do not stream -ENABLE_VIDEO_STREAMING=false -# URL of the video to stream. Can be a YouTube link or other video source. -VIDEO_URL=https://www.youtube.com/watch?v=rnXIjl_Rzy4 -# Port on which the video streaming server will run. -VIDEO_PORT=32800 -# Name of the video stream or service identifier. -VIDEO_NAME=sfstream -# Enable or disable object detections (like YOLO detections) on the video. -# true = enable detections, false = disable -ENABLE_DETECTIONS=false -# Port on which the YOLO model server will run for object detection. -YOLO_MODEL_PORT=50051 - -#--- MQTT --- -# Whether to enable the default MQTT process -ENABLE_MQTT=false - -# IP address of MQTT broker -MQTT_BROKER=172.104.228.251 -# Port associated with MQTT broker -MQTT_PORT=1883 -# Whether to enable MQTT logging process -MQTT_LOG=false - -# Topic to get data for -MSG_TOPIC=anylog-demo -# Logical database name -MSG_DBMS=mydb -# Table where to store data -MSG_TABLE=bring [table] -# Timestamp column name -MSG_TIMESTAMP_COLUMN=bring [timestamp] -# Value column name -MSG_VALUE_COLUMN=bring [value] -# Column value type -MSG_VALUE_COLUMN_TYPE=float - -#--- OPC-UA --- -# Whether or not to enable to OPC-UA service -ENABLE_OPCUA=false -# OPC-UA URL address (ex. opcua.tcp:;//127.0.0.1:4840) -OPCUA_URL="" -# Node information the root is located in (ex. ns=2;s=DataSet) -OPCUA_NODE="" -# How often to pull data from OPC-UA -OPCUA_FREQUENCY=5 - -#==== advanced ==== -# Fine-tuning for experienced users - safe to leave as defaults -#--- Directories --- -# AnyLog Root Path -ANYLOG_PATH=/app -# !local_scripts: ${ANYLOG_HOME}/deployment-scripts/scripts -LOCAL_SCRIPTS=/app/deployment-scripts -# !test_dir: ${ANYLOG_HOME}/deployment-scripts/test -TEST_DIR=/app/deployment-scripts/test - -#--- General --- -# hide blockchain policy - supports all node types except cluster / operator -IS_HIDDEN=false -# Whether to automatically run a local (or personalized) script at the end of the process -DEPLOY_LOCAL_SCRIPT=false -# Coordinates of the machine - used by Grafana to map the network -LOCATION="" -# Country where machine is located -COUNTRY="" -# State where machine is located -STATE="" -# City where machine is located -CITY="" - -#--- Networking --- -# Declare Policy name -CONFIG_NAME="" -# Whether to enable DNS resolution -ENABLE_DNS=false -# NIC type used to resolve the internal IP address (ex. eth0, lo) -NIC_TYPE="" -# Overlay IP address - if set, will replace local IP address when connecting to network -OVERLAY_IP="192.168.0.140" -# External DNS resolution -EXTERNAL_DNS="" -# local DNS domain to append to hostname (ex. example.com) -DNS_DOMAIN="" -# Full path for machine's (local) DNS -DNS="" - -# Bind settings - a true value binds to a specific IP and Port (a false value binds to all IPs) -TCP_BIND=false -REST_BIND=false -BROKER_BIND=false - -# Number of concurrent threads per connection protocol -TCP_THREADS=6 -REST_THREADS=6 -BROKER_THREADS=6 - -# Timeout in seconds for REST requests (0 = continuous) -REST_TIMEOUT=30 - -#--- NoSQL --- -# Reuse identical blobs using file hash -BLOBS_REUSE=true -# Compress blobs before storage -BLOBS_COMPRESS=true - -# Folder for storing blobs locally -# Leave empty to use default application path -BLOBS_FOLDER="" - -#--- Object Store --- -# Logical bucket grouping -BUCKET_GROUP="" -# Bucket identifier -BUCKET_ID="" -# Cloud region -BUCKET_REGION="" - -#--- Operator --- -# Operator ID - autogenerated INT if not specified -# This is used to help keep a record of where data was originally stored -MEMBER="" -# Whether of not to enable HA against the cluster -ENABLE_HA=false -# How many days back to sync between nodes -START_DATE=30 -# How many threads to use in the operator process -OPERATOR_THREADS=3 - -#--- Sub processes --- -# Compress JSON and SQL file(s) backup -COMPRESS_FILE=true -# When data comes in write to database immediately, as opposed to waiting for a full buffer -WRITE_IMMEDIATE=true -# If buffer is not full, how long to wait until pushing data through -THRESHOLD_TIME=1 seconds -# Buffer size to reach, at which point data is pushed through -THRESHOLD_VOLUME=10KB -# Number of parallel queries -QUERY_POOL=6 - -#==== secrets ==== -# Sensitive credentials - do not commit to version control -# will be updated to use sercets logic for both docker and k8s - -#--- General --- -# AnyLog License Key -LICENSE_KEY=5528032c1ee582225942bae2631c9adc97213543c9caec371e55c70c5a7c4ed2303cef87ab33e6499a42daf52cfb41a09b7f558334acf08c42d72f122f1cba88164ebeba48f176e2b261b7b8b34d0d404cd7397df7b80b16812706a01f632c6dc6ee27d1b69f39884b68c938242088cc60852103651c092f82e7ba52f4f41430{"company":"Roy","expiration":"2036-01-30","type":"beta"} - -#--- Database --- -# Username for SQL database connection -DB_USER=demo -# Password correlated to database user -DB_PASSWD=passwd - -#--- NoSQL --- -# Optional DB authentication -BLOB_STORAGE_USER=admin -BLOB_STORAGE_PASSWORD=passwd - -#--- MQTT --- -# User associated with MQTT broker -MQTT_USER=anyloguser -# Password associated with MQTT user -MQTT_PASSWD=mqtt4AnyLog! - -#--- Object Store --- -# Access credentials -BUCKET_ACCESS_KEY="" -BUCKET_SECRET_KEY="" - -#==== remote-gui ==== -# Companion UI application - deployed alongside the AnyLog node -# Whether to enable Remote-GUI in deployment -ENABLE_REMOTE_GUI=false -# Remote-GUI version Tag -REMOTE_GUI_TAG=beta2 -# NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 -REMOTE_GUI_NIC="" -# Remote GUI Frontend Port -REMOTE_GUI_FE=31800 -# Remote GUI Backend Port -REMOTE_GUI_BE=8080 -# Grafana PATH to reference for Grafana connection -GRAFANA_URL="" - - diff --git a/docker-makefiles/operator2/node_configs.env b/docker-makefiles/operator2/node_configs.env deleted file mode 100644 index 918df85c..00000000 --- a/docker-makefiles/operator2/node_configs.env +++ /dev/null @@ -1,289 +0,0 @@ -#==== .env ==== -# Basic deployment settings - -# Container image, deployment type -# -> prod: run AnyLog instance -# -> bash: run shell interface with AnyLog installed and params set -INIT_TYPE=prod - -# AnyLog image -IMAGE=anylogco/anylog-network - -# Github repo to use for configuring / deploying node -DEPLOYMENTS_REPO="https://github.com/AnyLog-co/deployment-scripts" -# Branch associated with git repo -DEPLOYMENTS_BRANCH="pre-develop" - -#==== basic ==== -# Core node configuration - required for all deployments -#--- General --- -# Configuration based deploymnet node deployment type -# Node Types: -# generic: sandbox with only network configured -# master: "blockchain" emulator -# operator: agent dedicated to storing data -# query: agent dedicated to querying data (enable system_query logical database) -# publisher: agent used to distribute data -# master-operator: a combination of master and operator on a single agent -# master-publisher: a combination of master and publisher on a single agent -NODE_TYPE=operator - -# Name of the AnyLog instance -NODE_NAME=operator2 -# Owner of the AnyLog instance -COMPANY_NAME=AnyLog Co. - -#--- Networking --- -# Port address used by AnyLog's TCP protocol to communicate with other nodes in the network -ANYLOG_SERVER_PORT=32248 -# Port address used by AnyLog's REST protocol -ANYLOG_REST_PORT=32249 -# Port value to be used as an MQTT broker, or some other third-party broker -ANYLOG_BROKER_PORT=32250 - -#--- Database --- -# Physical database type (sqlite, psql) -DB_TYPE=sqlite -# Database IP address -DB_IP=${INET_IP} -# Database port number -DB_PORT=5433 -# Whether to set autocommit data -AUTOCOMMIT=true - -#--- NoSQL --- -# Enable external blob storage -BLOBS_STORAGE=false -# Storage backend type - Supported: mongo, s3, akave, minio -BLOB_STORAGE_TYPE=mongo -# Storage endpoint (MongoDB or object storage endpoint) -BLOB_STORAGE_IP=${INET_IP} -# Storage port (used mainly for MongoDB) -BLOB_STORAGE_PORT=27017 - -#--- System Query --- -# Whether to start system_query logical database -SYSTEM_QUERY=true -# Database type used for system_query logical database -SYSTEM_QUERY_DB=sqlite -# Run system_query using in-memory SQLite. If set to false, will use pre-set database type -MEMORY=true - -#--- Blockchain --- -# TCP connection information for Master Node -LEDGER_CONN=${INET_IP}:32048 -# How often to sync from blockchain -BLOCKCHAIN_SYNC=10 second -# Source of where the blockchain data is coming from (master, optimism) -BLOCKCHAIN_SOURCE=master -# Where the local copy of the blockchain will be stored -BLOCKCHAIN_DESTINATION=file - -#--- Operator --- -# Owner of the cluster -CLUSTER_NAME=anylog-cluster2 -# Logical database name -DEFAULT_DBMS=mydb - -#==== southbound ==== -# Data ingestion protocols (MQTT, OPC-UA, video streaming, monitoring) -#--- Monitoring --- -# Whether to monitor the node or not -NODE_MONITORING=false -# Store monitoring in Operator node(s) -STORE_MONITORING=false -# Accept syslog data from local (Message broker required) -SYSLOG_MONITORING=false -# Docker Monitoring -DOCKER_MONITORING=false - -#--- Video Streaming --- -# Enable or disable video streaming. -# true = stream video, false = do not stream -ENABLE_VIDEO_STREAMING=false -# URL of the video to stream. Can be a YouTube link or other video source. -VIDEO_URL=https://www.youtube.com/watch?v=rnXIjl_Rzy4 -# Port on which the video streaming server will run. -VIDEO_PORT=32800 -# Name of the video stream or service identifier. -VIDEO_NAME=sfstream -# Enable or disable object detections (like YOLO detections) on the video. -# true = enable detections, false = disable -ENABLE_DETECTIONS=false -# Port on which the YOLO model server will run for object detection. -YOLO_MODEL_PORT=50051 - -#--- MQTT --- -# Whether to enable the default MQTT process -ENABLE_MQTT=false - -# IP address of MQTT broker -MQTT_BROKER=172.104.228.251 -# Port associated with MQTT broker -MQTT_PORT=1883 -# Whether to enable MQTT logging process -MQTT_LOG=false - -# Topic to get data for -MSG_TOPIC=anylog-demo -# Logical database name -MSG_DBMS=mydb -# Table where to store data -MSG_TABLE=bring [table] -# Timestamp column name -MSG_TIMESTAMP_COLUMN=bring [timestamp] -# Value column name -MSG_VALUE_COLUMN=bring [value] -# Column value type -MSG_VALUE_COLUMN_TYPE=float - -#--- OPC-UA --- -# Whether or not to enable to OPC-UA service -ENABLE_OPCUA=false -# OPC-UA URL address (ex. opcua.tcp:;//127.0.0.1:4840) -OPCUA_URL="" -# Node information the root is located in (ex. ns=2;s=DataSet) -OPCUA_NODE="" -# How often to pull data from OPC-UA -OPCUA_FREQUENCY=5 - -#==== advanced ==== -# Fine-tuning for experienced users - safe to leave as defaults -#--- Directories --- -# AnyLog Root Path -ANYLOG_PATH=/app -# !local_scripts: ${ANYLOG_HOME}/deployment-scripts/scripts -LOCAL_SCRIPTS=/app/deployment-scripts -# !test_dir: ${ANYLOG_HOME}/deployment-scripts/test -TEST_DIR=/app/deployment-scripts/test - -#--- General --- -# hide blockchain policy - supports all node types except cluster / operator -IS_HIDDEN=false -# Whether to automatically run a local (or personalized) script at the end of the process -DEPLOY_LOCAL_SCRIPT=false -# Coordinates of the machine - used by Grafana to map the network -LOCATION="" -# Country where machine is located -COUNTRY="" -# State where machine is located -STATE="" -# City where machine is located -CITY="" - -#--- Networking --- -# Declare Policy name -CONFIG_NAME="" -# Whether to enable DNS resolution -ENABLE_DNS=false -# NIC type used to resolve the internal IP address (ex. eth0, lo) -NIC_TYPE="" -# Overlay IP address - if set, will replace local IP address when connecting to network -OVERLAY_IP="192.168.0.140" -# External DNS resolution -EXTERNAL_DNS="" -# local DNS domain to append to hostname (ex. example.com) -DNS_DOMAIN="" -# Full path for machine's (local) DNS -DNS="" - -# Bind settings - a true value binds to a specific IP and Port (a false value binds to all IPs) -TCP_BIND=false -REST_BIND=false -BROKER_BIND=false - -# Number of concurrent threads per connection protocol -TCP_THREADS=6 -REST_THREADS=6 -BROKER_THREADS=6 - -# Timeout in seconds for REST requests (0 = continuous) -REST_TIMEOUT=30 - -#--- NoSQL --- -# Reuse identical blobs using file hash -BLOBS_REUSE=true -# Compress blobs before storage -BLOBS_COMPRESS=true - -# Folder for storing blobs locally -# Leave empty to use default application path -BLOBS_FOLDER="" - -#--- Object Store --- -# Logical bucket grouping -BUCKET_GROUP="" -# Bucket identifier -BUCKET_ID="" -# Cloud region -BUCKET_REGION="" - -#--- Operator --- -# Operator ID - autogenerated INT if not specified -# This is used to help keep a record of where data was originally stored -MEMBER="" -# Whether of not to enable HA against the cluster -ENABLE_HA=false -# How many days back to sync between nodes -START_DATE=30 -# How many threads to use in the operator process -OPERATOR_THREADS=3 - -#--- Sub processes --- -# Compress JSON and SQL file(s) backup -COMPRESS_FILE=true -# When data comes in write to database immediately, as opposed to waiting for a full buffer -WRITE_IMMEDIATE=true -# If buffer is not full, how long to wait until pushing data through -THRESHOLD_TIME=60 seconds -# Buffer size to reach, at which point data is pushed through -THRESHOLD_VOLUME=10KB -# Number of parallel queries -QUERY_POOL=6 - -#==== secrets ==== -# Sensitive credentials - do not commit to version control -# will be updated to use sercets logic for both docker and k8s - -#--- General --- -# AnyLog License Key -LICENSE_KEY=5528032c1ee582225942bae2631c9adc97213543c9caec371e55c70c5a7c4ed2303cef87ab33e6499a42daf52cfb41a09b7f558334acf08c42d72f122f1cba88164ebeba48f176e2b261b7b8b34d0d404cd7397df7b80b16812706a01f632c6dc6ee27d1b69f39884b68c938242088cc60852103651c092f82e7ba52f4f41430{"company":"Roy","expiration":"2036-01-30","type":"beta"} - -#--- Database --- -# Username for SQL database connection -DB_USER=demo -# Password correlated to database user -DB_PASSWD=passwd - -#--- NoSQL --- -# Optional DB authentication -BLOB_STORAGE_USER=admin -BLOB_STORAGE_PASSWORD=passwd - -#--- MQTT --- -# User associated with MQTT broker -MQTT_USER=anyloguser -# Password associated with MQTT user -MQTT_PASSWD=mqtt4AnyLog! - -#--- Object Store --- -# Access credentials -BUCKET_ACCESS_KEY="" -BUCKET_SECRET_KEY="" - -#==== remote-gui ==== -# Companion UI application - deployed alongside the AnyLog node -# Whether to enable Remote-GUI in deployment -ENABLE_REMOTE_GUI=false -# Remote-GUI version Tag -REMOTE_GUI_TAG=beta2 -# NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 -REMOTE_GUI_NIC="" -# Remote GUI Frontend Port -REMOTE_GUI_FE=31800 -# Remote GUI Backend Port -REMOTE_GUI_BE=8080 -# Grafana PATH to reference for Grafana connection -GRAFANA_URL="" - - diff --git a/docker-makefiles/operator3/node_configs.env b/docker-makefiles/operator3/node_configs.env deleted file mode 100644 index e11b7358..00000000 --- a/docker-makefiles/operator3/node_configs.env +++ /dev/null @@ -1,289 +0,0 @@ -#==== .env ==== -# Basic deployment settings - -# Container image, deployment type -# -> prod: run AnyLog instance -# -> bash: run shell interface with AnyLog installed and params set -INIT_TYPE=prod - -# AnyLog image -IMAGE=anylogco/anylog-network - -# Github repo to use for configuring / deploying node -DEPLOYMENTS_REPO="https://github.com/AnyLog-co/deployment-scripts" -# Branch associated with git repo -DEPLOYMENTS_BRANCH="pre-develop" - -#==== basic ==== -# Core node configuration - required for all deployments -#--- General --- -# Configuration based deploymnet node deployment type -# Node Types: -# generic: sandbox with only network configured -# master: "blockchain" emulator -# operator: agent dedicated to storing data -# query: agent dedicated to querying data (enable system_query logical database) -# publisher: agent used to distribute data -# master-operator: a combination of master and operator on a single agent -# master-publisher: a combination of master and publisher on a single agent -NODE_TYPE=operator - -# Name of the AnyLog instance -NODE_NAME=operator3 -# Owner of the AnyLog instance -COMPANY_NAME=AnyLog Co. - -#--- Networking --- -# Port address used by AnyLog's TCP protocol to communicate with other nodes in the network -ANYLOG_SERVER_PORT=32348 -# Port address used by AnyLog's REST protocol -ANYLOG_REST_PORT=32349 -# Port value to be used as an MQTT broker, or some other third-party broker -ANYLOG_BROKER_PORT=32350 - -#--- Database --- -# Physical database type (sqlite, psql) -DB_TYPE=sqlite -# Database IP address -DB_IP=${INET_IP} -# Database port number -DB_PORT=5434 -# Whether to set autocommit data -AUTOCOMMIT=true - -#--- NoSQL --- -# Enable external blob storage -BLOBS_STORAGE=false -# Storage backend type - Supported: mongo, s3, akave, minio -BLOB_STORAGE_TYPE=mongo -# Storage endpoint (MongoDB or object storage endpoint) -BLOB_STORAGE_IP=${INET_IP} -# Storage port (used mainly for MongoDB) -BLOB_STORAGE_PORT=27017 - -#--- System Query --- -# Whether to start system_query logical database -SYSTEM_QUERY=true -# Database type used for system_query logical database -SYSTEM_QUERY_DB=sqlite -# Run system_query using in-memory SQLite. If set to false, will use pre-set database type -MEMORY=true - -#--- Blockchain --- -# TCP connection information for Master Node -LEDGER_CONN=${INET_IP}:32048 -# How often to sync from blockchain -BLOCKCHAIN_SYNC=10 second -# Source of where the blockchain data is coming from (master, optimism) -BLOCKCHAIN_SOURCE=master -# Where the local copy of the blockchain will be stored -BLOCKCHAIN_DESTINATION=file - -#--- Operator --- -# Owner of the cluster -CLUSTER_NAME=anylog-cluster3 -# Logical database name -DEFAULT_DBMS=mydb - -#==== southbound ==== -# Data ingestion protocols (MQTT, OPC-UA, video streaming, monitoring) -#--- Monitoring --- -# Whether to monitor the node or not -NODE_MONITORING=false -# Store monitoring in Operator node(s) -STORE_MONITORING=false -# Accept syslog data from local (Message broker required) -SYSLOG_MONITORING=false -# Docker Monitoring -DOCKER_MONITORING=false - -#--- Video Streaming --- -# Enable or disable video streaming. -# true = stream video, false = do not stream -ENABLE_VIDEO_STREAMING=false -# URL of the video to stream. Can be a YouTube link or other video source. -VIDEO_URL=https://www.youtube.com/watch?v=rnXIjl_Rzy4 -# Port on which the video streaming server will run. -VIDEO_PORT=32800 -# Name of the video stream or service identifier. -VIDEO_NAME=sfstream -# Enable or disable object detections (like YOLO detections) on the video. -# true = enable detections, false = disable -ENABLE_DETECTIONS=false -# Port on which the YOLO model server will run for object detection. -YOLO_MODEL_PORT=50051 - -#--- MQTT --- -# Whether to enable the default MQTT process -ENABLE_MQTT=false - -# IP address of MQTT broker -MQTT_BROKER=172.104.228.251 -# Port associated with MQTT broker -MQTT_PORT=1883 -# Whether to enable MQTT logging process -MQTT_LOG=false - -# Topic to get data for -MSG_TOPIC=anylog-demo -# Logical database name -MSG_DBMS=mydb -# Table where to store data -MSG_TABLE=bring [table] -# Timestamp column name -MSG_TIMESTAMP_COLUMN=bring [timestamp] -# Value column name -MSG_VALUE_COLUMN=bring [value] -# Column value type -MSG_VALUE_COLUMN_TYPE=float - -#--- OPC-UA --- -# Whether or not to enable to OPC-UA service -ENABLE_OPCUA=false -# OPC-UA URL address (ex. opcua.tcp:;//127.0.0.1:4840) -OPCUA_URL="" -# Node information the root is located in (ex. ns=2;s=DataSet) -OPCUA_NODE="" -# How often to pull data from OPC-UA -OPCUA_FREQUENCY=5 - -#==== advanced ==== -# Fine-tuning for experienced users - safe to leave as defaults -#--- Directories --- -# AnyLog Root Path -ANYLOG_PATH=/app -# !local_scripts: ${ANYLOG_HOME}/deployment-scripts/scripts -LOCAL_SCRIPTS=/app/deployment-scripts -# !test_dir: ${ANYLOG_HOME}/deployment-scripts/test -TEST_DIR=/app/deployment-scripts/test - -#--- General --- -# hide blockchain policy - supports all node types except cluster / operator -IS_HIDDEN=false -# Whether to automatically run a local (or personalized) script at the end of the process -DEPLOY_LOCAL_SCRIPT=false -# Coordinates of the machine - used by Grafana to map the network -LOCATION="" -# Country where machine is located -COUNTRY="" -# State where machine is located -STATE="" -# City where machine is located -CITY="" - -#--- Networking --- -# Declare Policy name -CONFIG_NAME="" -# Whether to enable DNS resolution -ENABLE_DNS=false -# NIC type used to resolve the internal IP address (ex. eth0, lo) -NIC_TYPE="" -# Overlay IP address - if set, will replace local IP address when connecting to network -OVERLAY_IP="192.168.0.140" -# External DNS resolution -EXTERNAL_DNS="" -# local DNS domain to append to hostname (ex. example.com) -DNS_DOMAIN="" -# Full path for machine's (local) DNS -DNS="" - -# Bind settings - a true value binds to a specific IP and Port (a false value binds to all IPs) -TCP_BIND=false -REST_BIND=false -BROKER_BIND=false - -# Number of concurrent threads per connection protocol -TCP_THREADS=6 -REST_THREADS=6 -BROKER_THREADS=6 - -# Timeout in seconds for REST requests (0 = continuous) -REST_TIMEOUT=30 - -#--- NoSQL --- -# Reuse identical blobs using file hash -BLOBS_REUSE=true -# Compress blobs before storage -BLOBS_COMPRESS=true - -# Folder for storing blobs locally -# Leave empty to use default application path -BLOBS_FOLDER="" - -#--- Object Store --- -# Logical bucket grouping -BUCKET_GROUP="" -# Bucket identifier -BUCKET_ID="" -# Cloud region -BUCKET_REGION="" - -#--- Operator --- -# Operator ID - autogenerated INT if not specified -# This is used to help keep a record of where data was originally stored -MEMBER="" -# Whether of not to enable HA against the cluster -ENABLE_HA=false -# How many days back to sync between nodes -START_DATE=30 -# How many threads to use in the operator process -OPERATOR_THREADS=3 - -#--- Sub processes --- -# Compress JSON and SQL file(s) backup -COMPRESS_FILE=true -# When data comes in write to database immediately, as opposed to waiting for a full buffer -WRITE_IMMEDIATE=true -# If buffer is not full, how long to wait until pushing data through -THRESHOLD_TIME=10 seconds -# Buffer size to reach, at which point data is pushed through -THRESHOLD_VOLUME=10KB -# Number of parallel queries -QUERY_POOL=6 - -#==== secrets ==== -# Sensitive credentials - do not commit to version control -# will be updated to use sercets logic for both docker and k8s - -#--- General --- -# AnyLog License Key -LICENSE_KEY=5528032c1ee582225942bae2631c9adc97213543c9caec371e55c70c5a7c4ed2303cef87ab33e6499a42daf52cfb41a09b7f558334acf08c42d72f122f1cba88164ebeba48f176e2b261b7b8b34d0d404cd7397df7b80b16812706a01f632c6dc6ee27d1b69f39884b68c938242088cc60852103651c092f82e7ba52f4f41430{"company":"Roy","expiration":"2036-01-30","type":"beta"} - -#--- Database --- -# Username for SQL database connection -DB_USER=demo -# Password correlated to database user -DB_PASSWD=passwd - -#--- NoSQL --- -# Optional DB authentication -BLOB_STORAGE_USER=admin -BLOB_STORAGE_PASSWORD=passwd - -#--- MQTT --- -# User associated with MQTT broker -MQTT_USER=anyloguser -# Password associated with MQTT user -MQTT_PASSWD=mqtt4AnyLog! - -#--- Object Store --- -# Access credentials -BUCKET_ACCESS_KEY="" -BUCKET_SECRET_KEY="" - -#==== remote-gui ==== -# Companion UI application - deployed alongside the AnyLog node -# Whether to enable Remote-GUI in deployment -ENABLE_REMOTE_GUI=false -# Remote-GUI version Tag -REMOTE_GUI_TAG=beta2 -# NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 -REMOTE_GUI_NIC="" -# Remote GUI Frontend Port -REMOTE_GUI_FE=31800 -# Remote GUI Backend Port -REMOTE_GUI_BE=8080 -# Grafana PATH to reference for Grafana connection -GRAFANA_URL="" - - diff --git a/docker-makefiles/operator4/node_configs.env b/docker-makefiles/operator4/node_configs.env deleted file mode 100644 index 72a39895..00000000 --- a/docker-makefiles/operator4/node_configs.env +++ /dev/null @@ -1,289 +0,0 @@ -#==== .env ==== -# Basic deployment settings - -# Container image, deployment type -# -> prod: run AnyLog instance -# -> bash: run shell interface with AnyLog installed and params set -INIT_TYPE=prod - -# AnyLog image -IMAGE=anylogco/anylog-network - -# Github repo to use for configuring / deploying node -DEPLOYMENTS_REPO="https://github.com/AnyLog-co/deployment-scripts" -# Branch associated with git repo -DEPLOYMENTS_BRANCH="pre-develop" - -#==== basic ==== -# Core node configuration - required for all deployments -#--- General --- -# Configuration based deploymnet node deployment type -# Node Types: -# generic: sandbox with only network configured -# master: "blockchain" emulator -# operator: agent dedicated to storing data -# query: agent dedicated to querying data (enable system_query logical database) -# publisher: agent used to distribute data -# master-operator: a combination of master and operator on a single agent -# master-publisher: a combination of master and publisher on a single agent -NODE_TYPE=operator - -# Name of the AnyLog instance -NODE_NAME=operator4 -# Owner of the AnyLog instance -COMPANY_NAME=AnyLog Co. - -#--- Networking --- -# Port address used by AnyLog's TCP protocol to communicate with other nodes in the network -ANYLOG_SERVER_PORT=32548 -# Port address used by AnyLog's REST protocol -ANYLOG_REST_PORT=32549 -# Port value to be used as an MQTT broker, or some other third-party broker -ANYLOG_BROKER_PORT=32550 - -#--- Database --- -# Physical database type (sqlite, psql) -DB_TYPE=sqlite -# Database IP address -DB_IP=${INET_IP} -# Database port number -DB_PORT=5435 -# Whether to set autocommit data -AUTOCOMMIT=true - -#--- NoSQL --- -# Enable external blob storage -BLOBS_STORAGE=false -# Storage backend type - Supported: mongo, s3, akave, minio -BLOB_STORAGE_TYPE=mongo -# Storage endpoint (MongoDB or object storage endpoint) -BLOB_STORAGE_IP=${INET_IP} -# Storage port (used mainly for MongoDB) -BLOB_STORAGE_PORT=27017 - -#--- System Query --- -# Whether to start system_query logical database -SYSTEM_QUERY=true -# Database type used for system_query logical database -SYSTEM_QUERY_DB=sqlite -# Run system_query using in-memory SQLite. If set to false, will use pre-set database type -MEMORY=true - -#--- Blockchain --- -# TCP connection information for Master Node -LEDGER_CONN=${INET_IP}:32048 -# How often to sync from blockchain -BLOCKCHAIN_SYNC=10 second -# Source of where the blockchain data is coming from (master, optimism) -BLOCKCHAIN_SOURCE=master -# Where the local copy of the blockchain will be stored -BLOCKCHAIN_DESTINATION=file - -#--- Operator --- -# Owner of the cluster -CLUSTER_NAME=anylog-cluster4 -# Logical database name -DEFAULT_DBMS=mydb - -#==== southbound ==== -# Data ingestion protocols (MQTT, OPC-UA, video streaming, monitoring) -#--- Monitoring --- -# Whether to monitor the node or not -NODE_MONITORING=false -# Store monitoring in Operator node(s) -STORE_MONITORING=false -# Accept syslog data from local (Message broker required) -SYSLOG_MONITORING=false -# Docker Monitoring -DOCKER_MONITORING=false - -#--- Video Streaming --- -# Enable or disable video streaming. -# true = stream video, false = do not stream -ENABLE_VIDEO_STREAMING=false -# URL of the video to stream. Can be a YouTube link or other video source. -VIDEO_URL=https://www.youtube.com/watch?v=rnXIjl_Rzy4 -# Port on which the video streaming server will run. -VIDEO_PORT=32800 -# Name of the video stream or service identifier. -VIDEO_NAME=sfstream -# Enable or disable object detections (like YOLO detections) on the video. -# true = enable detections, false = disable -ENABLE_DETECTIONS=false -# Port on which the YOLO model server will run for object detection. -YOLO_MODEL_PORT=50051 - -#--- MQTT --- -# Whether to enable the default MQTT process -ENABLE_MQTT=false - -# IP address of MQTT broker -MQTT_BROKER=172.104.228.251 -# Port associated with MQTT broker -MQTT_PORT=1883 -# Whether to enable MQTT logging process -MQTT_LOG=false - -# Topic to get data for -MSG_TOPIC=anylog-demo -# Logical database name -MSG_DBMS=mydb -# Table where to store data -MSG_TABLE=bring [table] -# Timestamp column name -MSG_TIMESTAMP_COLUMN=bring [timestamp] -# Value column name -MSG_VALUE_COLUMN=bring [value] -# Column value type -MSG_VALUE_COLUMN_TYPE=float - -#--- OPC-UA --- -# Whether or not to enable to OPC-UA service -ENABLE_OPCUA=false -# OPC-UA URL address (ex. opcua.tcp:;//127.0.0.1:4840) -OPCUA_URL="" -# Node information the root is located in (ex. ns=2;s=DataSet) -OPCUA_NODE="" -# How often to pull data from OPC-UA -OPCUA_FREQUENCY=5 - -#==== advanced ==== -# Fine-tuning for experienced users - safe to leave as defaults -#--- Directories --- -# AnyLog Root Path -ANYLOG_PATH=/app -# !local_scripts: ${ANYLOG_HOME}/deployment-scripts/scripts -LOCAL_SCRIPTS=/app/deployment-scripts -# !test_dir: ${ANYLOG_HOME}/deployment-scripts/test -TEST_DIR=/app/deployment-scripts/test - -#--- General --- -# hide blockchain policy - supports all node types except cluster / operator -IS_HIDDEN=false -# Whether to automatically run a local (or personalized) script at the end of the process -DEPLOY_LOCAL_SCRIPT=false -# Coordinates of the machine - used by Grafana to map the network -LOCATION="" -# Country where machine is located -COUNTRY="" -# State where machine is located -STATE="" -# City where machine is located -CITY="" - -#--- Networking --- -# Declare Policy name -CONFIG_NAME="" -# Whether to enable DNS resolution -ENABLE_DNS=false -# NIC type used to resolve the internal IP address (ex. eth0, lo) -NIC_TYPE="" -# Overlay IP address - if set, will replace local IP address when connecting to network -OVERLAY_IP="" -# External DNS resolution -EXTERNAL_DNS="" -# local DNS domain to append to hostname (ex. example.com) -DNS_DOMAIN="" -# Full path for machine's (local) DNS -DNS="" - -# Bind settings - a true value binds to a specific IP and Port (a false value binds to all IPs) -TCP_BIND=false -REST_BIND=false -BROKER_BIND=false - -# Number of concurrent threads per connection protocol -TCP_THREADS=6 -REST_THREADS=6 -BROKER_THREADS=6 - -# Timeout in seconds for REST requests (0 = continuous) -REST_TIMEOUT=30 - -#--- NoSQL --- -# Reuse identical blobs using file hash -BLOBS_REUSE=true -# Compress blobs before storage -BLOBS_COMPRESS=true - -# Folder for storing blobs locally -# Leave empty to use default application path -BLOBS_FOLDER="" - -#--- Object Store --- -# Logical bucket grouping -BUCKET_GROUP="" -# Bucket identifier -BUCKET_ID="" -# Cloud region -BUCKET_REGION="" - -#--- Operator --- -# Operator ID - autogenerated INT if not specified -# This is used to help keep a record of where data was originally stored -MEMBER="" -# Whether of not to enable HA against the cluster -ENABLE_HA=false -# How many days back to sync between nodes -START_DATE=30 -# How many threads to use in the operator process -OPERATOR_THREADS=3 - -#--- Sub processes --- -# Compress JSON and SQL file(s) backup -COMPRESS_FILE=true -# When data comes in write to database immediately, as opposed to waiting for a full buffer -WRITE_IMMEDIATE=true -# If buffer is not full, how long to wait until pushing data through -THRESHOLD_TIME=10 seconds -# Buffer size to reach, at which point data is pushed through -THRESHOLD_VOLUME=10KB -# Number of parallel queries -QUERY_POOL=6 - -#==== secrets ==== -# Sensitive credentials - do not commit to version control -# will be updated to use sercets logic for both docker and k8s - -#--- General --- -# AnyLog License Key -LICENSE_KEY=5528032c1ee582225942bae2631c9adc97213543c9caec371e55c70c5a7c4ed2303cef87ab33e6499a42daf52cfb41a09b7f558334acf08c42d72f122f1cba88164ebeba48f176e2b261b7b8b34d0d404cd7397df7b80b16812706a01f632c6dc6ee27d1b69f39884b68c938242088cc60852103651c092f82e7ba52f4f41430{"company":"Roy","expiration":"2036-01-30","type":"beta"} - -#--- Database --- -# Username for SQL database connection -DB_USER=demo -# Password correlated to database user -DB_PASSWD=passwd - -#--- NoSQL --- -# Optional DB authentication -BLOB_STORAGE_USER=admin -BLOB_STORAGE_PASSWORD=passwd - -#--- MQTT --- -# User associated with MQTT broker -MQTT_USER=anyloguser -# Password associated with MQTT user -MQTT_PASSWD=mqtt4AnyLog! - -#--- Object Store --- -# Access credentials -BUCKET_ACCESS_KEY="" -BUCKET_SECRET_KEY="" - -#==== remote-gui ==== -# Companion UI application - deployed alongside the AnyLog node -# Whether to enable Remote-GUI in deployment -ENABLE_REMOTE_GUI=false -# Remote-GUI version Tag -REMOTE_GUI_TAG=beta2 -# NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 -REMOTE_GUI_NIC="" -# Remote GUI Frontend Port -REMOTE_GUI_FE=31800 -# Remote GUI Backend Port -REMOTE_GUI_BE=8080 -# Grafana PATH to reference for Grafana connection -GRAFANA_URL="" - - diff --git a/docker-makefiles/roy-master/node_configs.env b/docker-makefiles/roy-master/node_configs.env deleted file mode 100644 index 5949d8ef..00000000 --- a/docker-makefiles/roy-master/node_configs.env +++ /dev/null @@ -1,296 +0,0 @@ -#==== .env ==== -# Basic deployment settings - -# Container image, deployment type -# -> prod: run AnyLog instance -# -> bash: run shell interface with AnyLog installed and params set -INIT_TYPE=prod - -# AnyLog image -IMAGE=anylogco/anylog-network - -# Github repo to use for configuring / deploying node -DEPLOYMENTS_REPO="/mnt/c/Users/oshad/AnyLog-code/deployment-scripts" -# Branch associated with git repo -DEPLOYMENTS_BRANCH="os-dev" - -# specify the docker socket - to be used when deploying docker monitoring -DOCKER_SOCKET="/var/run/docker.sock" - -#==== basic ==== -# Core node configuration - required for all deployments -#--- General --- -# Configuration based deploymnet node deployment type -# Node Types: -# generic: sandbox with only network configured -# master: "blockchain" emulator -# operator: agent dedicated to storing data -# query: agent dedicated to querying data (enable system_query logical database) -# publisher: agent used to distribute data -# master-operator: a combination of master and operator on a single agent -# master-publisher: a combination of master and publisher on a single agent -NODE_TYPE=master - -# Name of the AnyLog instance -NODE_NAME=master -# Owner of the AnyLog instance -COMPANY_NAME=AnyLog Co. - -#--- Networking --- -# Port address used by AnyLog's TCP protocol to communicate with other nodes in the network -ANYLOG_SERVER_PORT=32048 -# Port address used by AnyLog's REST protocol -ANYLOG_REST_PORT=32049 -# Port value to be used as an MQTT broker, or some other third-party broker -ANYLOG_BROKER_PORT="" - -#--- Database --- -# Physical database type (sqlite, psql) -DB_TYPE=sqlite -# Database IP address -DB_IP=127.0.0.1 -# Database port number -DB_PORT=5432 -# Whether to set autocommit data -AUTOCOMMIT=true - -#--- NoSQL --- -# Enable external blob storage -BLOBS_STORAGE=false -# Storage backend type - Supported: mongo, s3, akave, minio -BLOB_STORAGE_TYPE=mongo -# Storage endpoint (MongoDB or object storage endpoint) -BLOB_STORAGE_IP=127.0.0.1 -# Storage port (used mainly for MongoDB) -BLOB_STORAGE_PORT=27017 - -#--- System Query --- -# Whether to start system_query logical database -SYSTEM_QUERY=false -# Database type used for system_query logical database -SYSTEM_QUERY_DB=sqlite -# Run system_query using in-memory SQLite. If set to false, will use pre-set database type -MEMORY=false - -#--- Blockchain --- -# TCP connection information for Master Node -LEDGER_CONN=127.0.0.1:32048 -# How often to sync from blockchain -BLOCKCHAIN_SYNC=60 second -# Source of where the blockchain data is coming from (master, optimism) -BLOCKCHAIN_SOURCE=master -# Where the local copy of the blockchain will be stored -BLOCKCHAIN_DESTINATION=file - -#--- Operator --- -# Owner of the cluster -CLUSTER_NAME=anylog-cluster1 -# Logical database name -DEFAULT_DBMS=mydb - -#==== southbound ==== -# Data ingestion protocols (MQTT, OPC-UA, video streaming, monitoring) -#--- Monitoring --- -# Whether to monitor the node or not -NODE_MONITORING=false -# Store monitoring in Operator node(s) -STORE_MONITORING=false -# Accept syslog data from local (Message broker required) -SYSLOG_MONITORING=false -# Docker Monitoring -DOCKER_MONITORING=false - -#--- Video Streaming --- -# Enable or disable video streaming. -# true = stream video, false = do not stream -ENABLE_VIDEO_STREAMING=false -# URL of the video to stream. Can be a YouTube link or other video source. -VIDEO_URL=https://www.youtube.com/watch?v=rnXIjl_Rzy4 -# Port on which the video streaming server will run. -VIDEO_PORT=32800 -# Name of the video stream or service identifier. -VIDEO_NAME=sfstream -# Enable or disable object detections (like YOLO detections) on the video. -# true = enable detections, false = disable -ENABLE_DETECTIONS=false -# Port on which the YOLO model server will run for object detection. -YOLO_MODEL_PORT=50051 - -#--- MQTT --- -# Whether to enable the default MQTT process -ENABLE_MQTT=false - -# IP address of MQTT broker -MQTT_BROKER=172.104.228.251 -# Port associated with MQTT broker -MQTT_PORT=1883 -# Whether to enable MQTT logging process -MQTT_LOG=false - -# Topic to get data for -MSG_TOPIC=anylog-demo -# Logical database name -MSG_DBMS=mydb -# Table where to store data -MSG_TABLE=bring [table] -# Timestamp column name -MSG_TIMESTAMP_COLUMN=bring [timestamp] -# Value column name -MSG_VALUE_COLUMN=bring [value] -# Column value type -MSG_VALUE_COLUMN_TYPE=float - -#--- OPC-UA --- -# Whether or not to enable to OPC-UA service -ENABLE_OPCUA=false -# OPC-UA URL address (ex. opcua.tcp:;//127.0.0.1:4840) -OPCUA_URL="" -# Node information the root is located in (ex. ns=2;s=DataSet) -OPCUA_NODE="" -# How often to pull data from OPC-UA -OPCUA_FREQUENCY=5 - -#==== advanced ==== -# Fine-tuning for experienced users - safe to leave as defaults -#--- Directories --- -# AnyLog Root Path -ANYLOG_PATH=/app -# !local_scripts: ${ANYLOG_HOME}/deployment-scripts/scripts -LOCAL_SCRIPTS=/app/deployment-scripts -# !test_dir: ${ANYLOG_HOME}/deployment-scripts/test -TEST_DIR=/app/deployment-scripts/test - -#--- General --- -# hide blockchain policy - supports all node types except cluster / operator -IS_HIDDEN=false -# Whether to automatically run a local (or personalized) script at the end of the process -DEPLOY_LOCAL_SCRIPT=false -# Coordinates of the machine - used by Grafana to map the network -LOCATION="" -# Country where machine is located -COUNTRY="" -# State where machine is located -STATE="" -# City where machine is located -CITY="" - -#--- Networking --- -# Declare Policy name -CONFIG_NAME="" -# Whether to enable DNS resolution -ENABLE_DNS=false -# NIC type used to resolve the internal IP address (ex. eth0, lo) -NIC_TYPE="" -# Overlay IP address - if set, will replace local IP address when connecting to network -OVERLAY_IP="" -# External DNS resolution -EXTERNAL_DNS="" -# local DNS domain to append to hostname (ex. example.com) -DNS_DOMAIN="" -# Full path for machine's (local) DNS -DNS="" - -# Bind settings - a true value binds to a specific IP and Port (a false value binds to all IPs) -TCP_BIND=false -REST_BIND=false -BROKER_BIND=false - -# Number of concurrent threads per connection protocol -TCP_THREADS=6 -REST_THREADS=6 -BROKER_THREADS=6 - -# Timeout in seconds for REST requests (0 = continuous) -REST_TIMEOUT=30 - -#--- NoSQL --- -# Reuse identical blobs using file hash -BLOBS_REUSE=true -# Compress blobs before storage -BLOBS_COMPRESS=true - -# Folder for storing blobs locally -# Leave empty to use default application path -BLOBS_FOLDER="" - -#--- Object Store --- -# Logical bucket grouping -BUCKET_GROUP="" -# Bucket identifier -BUCKET_ID="" -# Cloud region -BUCKET_REGION="" - -#--- Operator --- -# Operator ID - autogenerated INT if not specified -# This is used to help keep a record of where data was originally stored -MEMBER="" -# Whether of not to enable HA against the cluster -ENABLE_HA=false -# How many days back to sync between nodes -START_DATE=30 -# How many threads to use in the operator process -OPERATOR_THREADS=3 - -#--- Sub processes --- -# Compress JSON and SQL file(s) backup -COMPRESS_FILE=true -# When data comes in write to database immediately, as opposed to waiting for a full buffer -WRITE_IMMEDIATE=true -# If buffer is not full, how long to wait until pushing data through -THRESHOLD_TIME=60 seconds -# Buffer size to reach, at which point data is pushed through -THRESHOLD_VOLUME=10KB -# Number of parallel queries -QUERY_POOL=6 - -#--- DEBUG --- -ENABLE_TRACEBACK=false -TRACE_LEVEL=0 - -#==== secrets ==== -# Sensitive credentials - do not commit to version control -# will be updated to use secrets logic for both docker and k8s - -#--- General --- -# AnyLog License Key -LICENSE_KEY=5528032c1ee582225942bae2631c9adc97213543c9caec371e55c70c5a7c4ed2303cef87ab33e6499a42daf52cfb41a09b7f558334acf08c42d72f122f1cba88164ebeba48f176e2b261b7b8b34d0d404cd7397df7b80b16812706a01f632c6dc6ee27d1b69f39884b68c938242088cc60852103651c092f82e7ba52f4f41430{"company":"Roy","expiration":"2036-01-30","type":"beta"} - -#--- Database --- -# Username for SQL database connection -DB_USER=admin -# Password correlated to database user -DB_PASSWD=passwd - -#--- NoSQL --- -# Optional DB authentication -BLOB_STORAGE_USER=admin -BLOB_STORAGE_PASSWORD=passwd - -#--- MQTT --- -# User associated with MQTT broker -MQTT_USER=anyloguser -# Password associated with MQTT user -MQTT_PASSWD=mqtt4AnyLog! - -#--- Object Store --- -# Access credentials -BUCKET_ACCESS_KEY="" -BUCKET_SECRET_KEY="" - -#==== remote-gui ==== -# Companion UI application - deployed alongside the AnyLog node -# Whether to enable Remote-GUI in deployment -ENABLE_REMOTE_GUI=false -# Remote-GUI version Tag -REMOTE_GUI_TAG=beta2 -# NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 -REMOTE_GUI_NIC="" -# Remote GUI Frontend Port -REMOTE_GUI_FE=31800 -# Remote GUI Backend Port -REMOTE_GUI_BE=8080 -# Grafana PATH to reference for Grafana connection -GRAFANA_URL="" - - diff --git a/docker-makefiles/roy-operator1/node_configs.env b/docker-makefiles/roy-operator1/node_configs.env deleted file mode 100644 index de17ec34..00000000 --- a/docker-makefiles/roy-operator1/node_configs.env +++ /dev/null @@ -1,296 +0,0 @@ -#==== .env ==== -# Basic deployment settings - -# Container image, deployment type -# -> prod: run AnyLog instance -# -> bash: run shell interface with AnyLog installed and params set -INIT_TYPE=prod - -# AnyLog image -IMAGE=anylogco/anylog-network - -# Github repo to use for configuring / deploying node -DEPLOYMENTS_REPO="/mnt/c/Users/oshad/AnyLog-code/deployment-scripts" -# Branch associated with git repo -DEPLOYMENTS_BRANCH="os-dev" - -# specify the docker socket - to be used when deploying docker monitoring -DOCKER_SOCKET="/var/run/docker.sock" - -#==== basic ==== -# Core node configuration - required for all deployments -#--- General --- -# Configuration based deploymnet node deployment type -# Node Types: -# generic: sandbox with only network configured -# master: "blockchain" emulator -# operator: agent dedicated to storing data -# query: agent dedicated to querying data (enable system_query logical database) -# publisher: agent used to distribute data -# master-operator: a combination of master and operator on a single agent -# master-publisher: a combination of master and publisher on a single agent -NODE_TYPE=operator - -# Name of the AnyLog instance -NODE_NAME=operator1 -# Owner of the AnyLog instance -COMPANY_NAME=AnyLog Co. - -#--- Networking --- -# Port address used by AnyLog's TCP protocol to communicate with other nodes in the network -ANYLOG_SERVER_PORT=32148 -# Port address used by AnyLog's REST protocol -ANYLOG_REST_PORT=32149 -# Port value to be used as an MQTT broker, or some other third-party broker -ANYLOG_BROKER_PORT=32150 - -#--- Database --- -# Physical database type (sqlite, psql) -DB_TYPE=sqlite -# Database IP address -DB_IP=${INET_IP} -# Database port number -DB_PORT=5432 -# Whether to set autocommit data -AUTOCOMMIT=true - -#--- NoSQL --- -# Enable external blob storage -BLOBS_STORAGE=false -# Storage backend type - Supported: mongo, s3, akave, minio -BLOB_STORAGE_TYPE=mongo -# Storage endpoint (MongoDB or object storage endpoint) -BLOB_STORAGE_IP=${INET_IP} -# Storage port (used mainly for MongoDB) -BLOB_STORAGE_PORT=27017 - -#--- System Query --- -# Whether to start system_query logical database -SYSTEM_QUERY=True -# Database type used for system_query logical database -SYSTEM_QUERY_DB=sqlite -# Run system_query using in-memory SQLite. If set to false, will use pre-set database type -MEMORY=True - -#--- Blockchain --- -# TCP connection information for Master Node -LEDGER_CONN=${INET_IP}:32048 -# How often to sync from blockchain -BLOCKCHAIN_SYNC=60 second -# Source of where the blockchain data is coming from (master, optimism) -BLOCKCHAIN_SOURCE=master -# Where the local copy of the blockchain will be stored -BLOCKCHAIN_DESTINATION=file - -#--- Operator --- -# Owner of the cluster -CLUSTER_NAME=anylog-cluster1 -# Logical database name -DEFAULT_DBMS=mydb - -#==== southbound ==== -# Data ingestion protocols (MQTT, OPC-UA, video streaming, monitoring) -#--- Monitoring --- -# Whether to monitor the node or not -NODE_MONITORING=false -# Store monitoring in Operator node(s) -STORE_MONITORING=false -# Accept syslog data from local (Message broker required) -SYSLOG_MONITORING=false -# Docker Monitoring -DOCKER_MONITORING=false - -#--- Video Streaming --- -# Enable or disable video streaming. -# true = stream video, false = do not stream -ENABLE_VIDEO_STREAMING=false -# URL of the video to stream. Can be a YouTube link or other video source. -VIDEO_URL=https://www.youtube.com/watch?v=rnXIjl_Rzy4 -# Port on which the video streaming server will run. -VIDEO_PORT=32800 -# Name of the video stream or service identifier. -VIDEO_NAME=sfstream -# Enable or disable object detections (like YOLO detections) on the video. -# true = enable detections, false = disable -ENABLE_DETECTIONS=false -# Port on which the YOLO model server will run for object detection. -YOLO_MODEL_PORT=50051 - -#--- MQTT --- -# Whether to enable the default MQTT process -ENABLE_MQTT=false - -# IP address of MQTT broker -MQTT_BROKER=172.104.228.251 -# Port associated with MQTT broker -MQTT_PORT=1883 -# Whether to enable MQTT logging process -MQTT_LOG=false - -# Topic to get data for -MSG_TOPIC=anylog-demo -# Logical database name -MSG_DBMS=mydb -# Table where to store data -MSG_TABLE=bring [table] -# Timestamp column name -MSG_TIMESTAMP_COLUMN=bring [timestamp] -# Value column name -MSG_VALUE_COLUMN=bring [value] -# Column value type -MSG_VALUE_COLUMN_TYPE=float - -#--- OPC-UA --- -# Whether or not to enable to OPC-UA service -ENABLE_OPCUA=false -# OPC-UA URL address (ex. opcua.tcp:;//127.0.0.1:4840) -OPCUA_URL="" -# Node information the root is located in (ex. ns=2;s=DataSet) -OPCUA_NODE="" -# How often to pull data from OPC-UA -OPCUA_FREQUENCY=5 - -#==== advanced ==== -# Fine-tuning for experienced users - safe to leave as defaults -#--- Directories --- -# AnyLog Root Path -ANYLOG_PATH=/app -# !local_scripts: ${ANYLOG_HOME}/deployment-scripts/scripts -LOCAL_SCRIPTS=/app/deployment-scripts -# !test_dir: ${ANYLOG_HOME}/deployment-scripts/test -TEST_DIR=/app/deployment-scripts/test - -#--- General --- -# hide blockchain policy - supports all node types except cluster / operator -IS_HIDDEN=false -# Whether to automatically run a local (or personalized) script at the end of the process -DEPLOY_LOCAL_SCRIPT=false -# Coordinates of the machine - used by Grafana to map the network -LOCATION="" -# Country where machine is located -COUNTRY="" -# State where machine is located -STATE="" -# City where machine is located -CITY="" - -#--- Networking --- -# Declare Policy name -CONFIG_NAME="" -# Whether to enable DNS resolution -ENABLE_DNS=false -# NIC type used to resolve the internal IP address (ex. eth0, lo) -NIC_TYPE="" -# Overlay IP address - if set, will replace local IP address when connecting to network -OVERLAY_IP="" -# External DNS resolution -EXTERNAL_DNS="" -# local DNS domain to append to hostname (ex. example.com) -DNS_DOMAIN="" -# Full path for machine's (local) DNS -DNS="" - -# Bind settings - a true value binds to a specific IP and Port (a false value binds to all IPs) -TCP_BIND=false -REST_BIND=false -BROKER_BIND=false - -# Number of concurrent threads per connection protocol -TCP_THREADS=6 -REST_THREADS=6 -BROKER_THREADS=6 - -# Timeout in seconds for REST requests (0 = continuous) -REST_TIMEOUT=30 - -#--- NoSQL --- -# Reuse identical blobs using file hash -BLOBS_REUSE=true -# Compress blobs before storage -BLOBS_COMPRESS=true - -# Folder for storing blobs locally -# Leave empty to use default application path -BLOBS_FOLDER="" - -#--- Object Store --- -# Logical bucket grouping -BUCKET_GROUP="" -# Bucket identifier -BUCKET_ID="" -# Cloud region -BUCKET_REGION="" - -#--- Operator --- -# Operator ID - autogenerated INT if not specified -# This is used to help keep a record of where data was originally stored -MEMBER="" -# Whether of not to enable HA against the cluster -ENABLE_HA=false -# How many days back to sync between nodes -START_DATE=30 -# How many threads to use in the operator process -OPERATOR_THREADS=3 - -#--- Sub processes --- -# Compress JSON and SQL file(s) backup -COMPRESS_FILE=true -# When data comes in write to database immediately, as opposed to waiting for a full buffer -WRITE_IMMEDIATE=true -# If buffer is not full, how long to wait until pushing data through -THRESHOLD_TIME=1 seconds -# Buffer size to reach, at which point data is pushed through -THRESHOLD_VOLUME=10KB -# Number of parallel queries -QUERY_POOL=6 - -#--- DEBUG --- -ENABLE_TRACEBACK=false -TRACE_LEVEL=0 - -#==== secrets ==== -# Sensitive credentials - do not commit to version control -# will be updated to use sercets logic for both docker and k8s - -#--- General --- -# AnyLog License Key -LICENSE_KEY=5528032c1ee582225942bae2631c9adc97213543c9caec371e55c70c5a7c4ed2303cef87ab33e6499a42daf52cfb41a09b7f558334acf08c42d72f122f1cba88164ebeba48f176e2b261b7b8b34d0d404cd7397df7b80b16812706a01f632c6dc6ee27d1b69f39884b68c938242088cc60852103651c092f82e7ba52f4f41430{"company":"Roy","expiration":"2036-01-30","type":"beta"} - -#--- Database --- -# Username for SQL database connection -DB_USER=demo -# Password correlated to database user -DB_PASSWD=passwd - -#--- NoSQL --- -# Optional DB authentication -BLOB_STORAGE_USER=admin -BLOB_STORAGE_PASSWORD=passwd - -#--- MQTT --- -# User associated with MQTT broker -MQTT_USER=anyloguser -# Password associated with MQTT user -MQTT_PASSWD=mqtt4AnyLog! - -#--- Object Store --- -# Access credentials -BUCKET_ACCESS_KEY="" -BUCKET_SECRET_KEY="" - -#==== remote-gui ==== -# Companion UI application - deployed alongside the AnyLog node -# Whether to enable Remote-GUI in deployment -ENABLE_REMOTE_GUI=false -# Remote-GUI version Tag -REMOTE_GUI_TAG=beta2 -# NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 -REMOTE_GUI_NIC="" -# Remote GUI Frontend Port -REMOTE_GUI_FE=31800 -# Remote GUI Backend Port -REMOTE_GUI_BE=8080 -# Grafana PATH to reference for Grafana connection -GRAFANA_URL="" - - From 9be036517a1028acdb4b655464c2897ad6475a2a Mon Sep 17 00:00:00 2001 From: Ori Shadmon Date: Fri, 8 May 2026 16:37:57 -0700 Subject: [PATCH 15/46] in progress - integration of license from Mark Signed-off-by: Ori Shadmon --- Makefile | 198 +-------------- license-key-generator/deploy.sh | 420 ++++++++++++++++++++++++++++++++ 2 files changed, 421 insertions(+), 197 deletions(-) create mode 100644 license-key-generator/deploy.sh diff --git a/Makefile b/Makefile index e6c3538d..9dd4fe6b 100644 --- a/Makefile +++ b/Makefile @@ -33,204 +33,8 @@ ifneq ($(TEST_CONN),) _FLAGS += --test-conn $(TEST_CONN) endif -ANYLOG_SH := bash deploy.sh +ANYLOG_SH := bash ./docker-makefiles/deploy.sh -# ────────────────────────────────────────────── -# License acceptance endpoint -# ────────────────────────────────────────────── -LICENSE_URL ?= http://23.239.12.151:8001/api/license-accept - -# ────────────────────────────────────────────── -all: help - -# ────────────────────────────────────────────── -# License acceptance -# ────────────────────────────────────────────── -license-check: .license_accepted ## accept license agreement (auto-runs before up/pull) - -.license_accepted: - @more LICENSE.txt - @echo "" - @echo "========================================" - @echo " License Agreement Acceptance Required" - @echo "========================================" - @echo "" - @while true; do \ - printf "Full Name: "; read NAME; \ - echo "$$NAME" | grep -qE '^[A-Za-z]{2,}( [A-Za-z]+)+$$' && break; \ - echo " ERROR: Please enter a valid full name (first and last name, letters only)."; \ - done; \ - while true; do \ - printf "Company: "; read COMPANY; \ - [ $$(echo "$$COMPANY" | tr -d '[:space:]' | wc -c) -ge 2 ] && break; \ - echo " ERROR: Company name must be at least 2 characters."; \ - done; \ - while true; do \ - printf "Email: "; read EMAIL; \ - echo "$$EMAIL" | grep -qE '^[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}$$' && break; \ - echo " ERROR: Please enter a valid email address (e.g. user@example.com)."; \ - done; \ - while true; do \ - printf "Project: "; read PROJECT; \ - [ $$(echo "$$PROJECT" | tr -d '[:space:]' | wc -c) -ge 2 ] && break; \ - echo " ERROR: Project name must be at least 2 characters."; \ - done; \ - while true; do \ - printf "License Key: "; read LICENSE_KEY; \ - _LQ=$$(printf '\342\200\234'); \ - _RQ=$$(printf '\342\200\235'); \ - _RESULT=$$(echo "$$LICENSE_KEY" | awk -v lq="$$_LQ" -v rq="$$_RQ" '{ \ - brace = index($$0, "{"); \ - if (brace == 0) { print "INVALID|no brace in key"; exit } \ - hex = substr($$0, 1, brace - 1); \ - json = substr($$0, brace); \ - gsub(/\\"/, "\"", json); \ - gsub(lq, "\"", json); \ - gsub(rq, "\"", json); \ - hex_ok = (length(hex) == 256 && hex ~ /^[0-9a-f]/); \ - co_ok = (json ~ /"company":"[^"]+"/); \ - ex_ok = (json ~ /"expiration":"[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]"/); \ - ty_ok = (json ~ /"type":"[^"]+"/); \ - if (hex_ok && co_ok && ex_ok && ty_ok) { print "VALID"; exit } \ - msg = "INVALID|"; \ - if (!hex_ok) msg = msg "hex must be 256 lowercase hex chars (got " length(hex) "); "; \ - if (!co_ok) msg = msg "missing/invalid company; "; \ - if (!ex_ok) msg = msg "missing/invalid expiration YYYY-MM-DD; "; \ - if (!ty_ok) msg = msg "missing/invalid type; "; \ - print msg; exit \ - }'); \ - if [ "$$_RESULT" = "VALID" ]; then \ - LICENSE_KEY=$$(echo "$$LICENSE_KEY" | awk -v lq="$$_LQ" -v rq="$$_RQ" '{gsub(/\\"/, "\""); gsub(lq, "\""); gsub(rq, "\""); print}'); \ - break; \ - fi; \ - echo " ERROR: Invalid license key - $$(echo "$$_RESULT" | cut -d'|' -f2)"; \ - done; \ - echo ""; \ - while true; do \ - printf "Do you accept the license agreement? [yes/no]: "; read ANSWER; \ - case "$$ANSWER" in \ - yes|no) break ;; \ - *) echo " ERROR: Please type 'yes' or 'no'." ;; \ - esac; \ - done; \ - echo ""; \ - if [ "$$ANSWER" != "yes" ]; then \ - echo "License not accepted. Aborting."; \ - exit 1; \ - fi; \ - echo "Registering license acceptance..."; \ - python3 -c "import json,sys;print(json.dumps({'name':sys.argv[1],'company':sys.argv[2],'email':sys.argv[3],'project':sys.argv[4],'license_key':sys.argv[5],'timestamp':sys.argv[6]}))" "$$NAME" "$$COMPANY" "$$EMAIL" "$$PROJECT" "$$LICENSE_KEY" "$$(date -u +%Y-%m-%dT%H:%M:%SZ)" > /tmp/license_payload.json; \ - post_license_payload() { \ - _URL="$$1"; _PAYLOAD_FILE="$$2"; _RESPONSE_FILE="$$3"; \ - if command -v curl >/dev/null 2>&1; then \ - curl -sk -o "$$_RESPONSE_FILE" -w "%{http_code}" \ - -X POST "$$_URL" \ - -H "Content-Type: application/json" \ - -d @"$$_PAYLOAD_FILE"; \ - return; \ - fi; \ - case "$$_URL" in \ - http://*) _REST="$${_URL#http://}"; _PORT=80 ;; \ - https://*) echo "ERROR: curl is not installed and Bash /dev/tcp cannot do HTTPS/TLS by itself." >&2; return 2 ;; \ - *) echo "ERROR: LICENSE_URL must start with http:// when curl is unavailable." >&2; return 2 ;; \ - esac; \ - _HOSTPORT="$${_REST%%/*}"; _PATH="/$${_REST#*/}"; \ - if [ "$$_HOSTPORT" = "$$_REST" ]; then _PATH="/"; fi; \ - _HOST="$${_HOSTPORT%%:*}"; \ - if [ "$$_HOST" != "$$_HOSTPORT" ]; then _PORT="$${_HOSTPORT##*:}"; fi; \ - _BODY=$$(<"$$_PAYLOAD_FILE"); \ - : > "$$_RESPONSE_FILE"; \ - if ! exec 3<>"/dev/tcp/$$_HOST/$$_PORT"; then \ - echo "ERROR: could not connect to $$_HOST:$$_PORT" >&2; \ - return 2; \ - fi; \ - printf 'POST %s HTTP/1.1\r\n' "$$_PATH" >&3; \ - printf 'Host: %s\r\n' "$$_HOSTPORT" >&3; \ - printf 'Content-Type: application/json\r\n' >&3; \ - printf 'Content-Length: %d\r\n' "$${#_BODY}" >&3; \ - printf 'Connection: close\r\n' >&3; \ - printf '\r\n' >&3; \ - printf '%s' "$$_BODY" >&3; \ - while IFS= read -r _LINE <&3; do printf '%s\n' "$$_LINE" >> "$$_RESPONSE_FILE"; done; \ - exec 3<&-; exec 3>&-; \ - IFS=' ' read -r _HTTP _CODE _MSG < "$$_RESPONSE_FILE"; \ - printf '%s' "$$_CODE"; \ - }; \ - HTTP_CODE=$$(post_license_payload "$(LICENSE_URL)" /tmp/license_payload.json /tmp/license_response.txt); \ - if [ "$$HTTP_CODE" = "200" ] || [ "$$HTTP_CODE" = "201" ]; then \ - echo "Registration successful."; \ - echo "$$NAME|$$COMPANY|$$EMAIL|$$PROJECT|$$LICENSE_KEY|$$(date -u +%Y-%m-%dT%H:%M:%SZ)" > .license_accepted; \ - else \ - echo "We could not complete license registration right now."; \ - echo "Please contact support@anylog.co with your license information."; \ - printf "Continue anyway? [yes/no]: "; read CONT; \ - if [ "$$CONT" != "yes" ]; then \ - echo "Aborting."; exit 1; \ - fi; \ - echo "$$NAME|$$COMPANY|$$EMAIL|$$PROJECT|$$LICENSE_KEY|$$(date -u +%Y-%m-%dT%H:%M:%SZ)|unregistered" > .license_accepted; \ - fi - -clean-license: ## reset license acceptance (re-prompts on next up/pull) - @rm -f .license_accepted - @echo "License acceptance cleared." - -debug-key: ## diagnose license key input (run this to troubleshoot key validation) - @printf "Paste license key: "; read K; \ - echo "--- od (raw bytes around brace) ---"; \ - echo "$$K" | od -c | grep -A2 '{'; \ - echo "--- smart quote check ---"; \ - echo "$$K" | od -c | grep -q '342 200 234' && echo "WARNING: left smart-quotes detected" || echo "no left smart-quotes"; \ - echo "$$K" | od -c | grep -q '342 200 235' && echo "WARNING: right smart-quotes detected" || echo "no right smart-quotes"; \ - echo "--- awk parse ---"; \ - _LQ=$$(printf '\342\200\234'); \ - _RQ=$$(printf '\342\200\235'); \ - echo "$$K" | awk -v lq="$$_LQ" -v rq="$$_RQ" '{ \ - brace = index($$0, "{"); \ - print "brace_pos=" brace; \ - print "hex_len=" (brace > 1 ? brace-1 : 0); \ - json = substr($$0, brace); \ - print "json_raw=[" json "]"; \ - gsub(/\\"/, "\"", json); \ - gsub(lq, "\"", json); \ - gsub(rq, "\"", json); \ - print "json_clean=[" json "]"; \ - print "co_ok=" (json ~ /"company":"[^"]+"/); \ - print "ex_ok=" (json ~ /"expiration":"[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]"/); \ - print "ty_ok=" (json ~ /"type":"[^"]+"/); \ - }' - -debug-post: ## test POST to license server and show exact request/response - @echo "--- testing POST to $(LICENSE_URL) ---"; \ - BODY="{\"name\":\"Test User\",\"company\":\"Test Co\",\"email\":\"test@test.com\",\"project\":\"test\",\"license_key\":\"testkey\",\"timestamp\":\"$$(date -u +%Y-%m-%dT%H:%M:%SZ)\"}"; \ - echo "Request body:"; \ - echo "$$BODY"; \ - echo ""; \ - echo "Response:"; \ - if command -v curl >/dev/null 2>&1; then \ - curl -sk -v -X POST "$(LICENSE_URL)" \ - -H "Content-Type: application/json" \ - -d "$$BODY" 2>&1; \ - else \ - _URL="$(LICENSE_URL)"; \ - case "$$_URL" in \ - http://*) _REST="$${_URL#http://}"; _PORT=80 ;; \ - https://*) echo "ERROR: curl is not installed and Bash /dev/tcp cannot do HTTPS/TLS by itself."; exit 2 ;; \ - *) echo "ERROR: LICENSE_URL must start with http:// when curl is unavailable."; exit 2 ;; \ - esac; \ - _HOSTPORT="$${_REST%%/*}"; _PATH="/$${_REST#*/}"; \ - if [ "$$_HOSTPORT" = "$$_REST" ]; then _PATH="/"; fi; \ - _HOST="$${_HOSTPORT%%:*}"; \ - if [ "$$_HOST" != "$$_HOSTPORT" ]; then _PORT="$${_HOSTPORT##*:}"; fi; \ - exec 3<>"/dev/tcp/$$_HOST/$$_PORT"; \ - printf 'POST %s HTTP/1.1\r\n' "$$_PATH" >&3; \ - printf 'Host: %s\r\n' "$$_HOSTPORT" >&3; \ - printf 'Content-Type: application/json\r\n' >&3; \ - printf 'Content-Length: %d\r\n' "$${#BODY}" >&3; \ - printf 'Connection: close\r\n\r\n' >&3; \ - printf '%s' "$$BODY" >&3; \ - cat <&3; \ - exec 3<&-; exec 3>&-; \ - fi # ────────────────────────────────────────────── # Docker Hub diff --git a/license-key-generator/deploy.sh b/license-key-generator/deploy.sh new file mode 100644 index 00000000..8efb2985 --- /dev/null +++ b/license-key-generator/deploy.sh @@ -0,0 +1,420 @@ +#!/usr/bin/env bash +# deploy.sh — AnyLog node lifecycle manager +# Usage: bash deploy.sh [OPTIONS] +# Run: bash deploy.sh help +set -euo pipefail + +# ────────────────────────────────────────────── +# Defaults (override via environment or flags) +# ────────────────────────────────────────────── +export LICENSE_KEY="${LICENSE_KEY:-}" + +# ────────────────────────────────────────────── +# Helpers +# ────────────────────────────────────────────── +die() { echo "ERROR: $1" >&2; exit "${2:-1}"; } + +# Resolve short-form aliases (operator → anylog-operator) +_resolve_type() { + case "$ANYLOG_TYPE" in + generic|master|operator|query|publisher|standalone-operator|standalone-publisher) + ANYLOG_TYPE="anylog-${ANYLOG_TYPE}" ;; + esac +} + +# Detect container runtime +_detect_runtime() { + if command -v podman >/dev/null 2>&1; then + CONTAINER_CMD="podman" + else + CONTAINER_CMD="docker" + fi + + if command -v podman-compose >/dev/null 2>&1; then + DOCKER_COMPOSE_CMD="podman-compose" + elif command -v docker-compose >/dev/null 2>&1; then + DOCKER_COMPOSE_CMD="docker-compose" + else + DOCKER_COMPOSE_CMD="docker compose" + fi +} + +# Detect platform +_detect_platform() { + UNAME_M=$(uname -m) + case "$UNAME_M" in + x86_64) DOCKER_PLATFORM="linux/amd64" ;; + aarch64|arm64) DOCKER_PLATFORM="linux/arm64" ;; + *) die "Unsupported architecture: ${UNAME_M}" ;; + esac + export DOCKER_PLATFORM + export ANYLOG_UID=$(id -u) + export ANYLOG_GID=$(id -g) +} + +# Load IMAGE and NODE_NAME from config files +_load_configs() { + local env_file="docker-makefiles/${ANYLOG_TYPE}/.env" + local single_file="docker-makefiles/${ANYLOG_TYPE}/node_configs.env" + + if [[ -f "$env_file" ]]; then + IMAGE=$(grep -m1 '^IMAGE=' "$env_file" | cut -d= -f2- | tr -d '"\r') + NODE_NAME=$(grep -m1 '^NODE_NAME=' "$env_file" | cut -d= -f2- | tr -d '"\r') + elif [[ -f "$single_file" ]]; then + IMAGE=$(grep -m1 '^IMAGE=' "$single_file" | cut -d= -f2- | tr -d '"\r') + NODE_NAME=$(grep -m1 '^NODE_NAME=' "$single_file" | cut -d= -f2- | tr -d '"\r') + else + die "Missing configuration file(s) for '${ANYLOG_TYPE}'" + fi +} + +# Resolve TEST_CONN if not set +_resolve_test_conn() { + if [[ -z "$TEST_CONN" ]]; then + local single_file="docker-makefiles/${ANYLOG_TYPE}/node_configs.env" + local rest_port + rest_port=$(grep -m1 '^ANYLOG_REST_PORT=' "$single_file" 2>/dev/null | cut -d= -f2- | tr -d '"\r' || echo "32549") + TEST_CONN="127.0.0.1:${rest_port}" + fi +} + +# Validate that ANYLOG_TYPE directory exists +_check_configs() { + [[ -n "$ANYLOG_TYPE" ]] || die "ANYLOG_TYPE is required" + [[ -d "docker-makefiles/${ANYLOG_TYPE}" ]] || \ + die "Missing directory 'docker-makefiles/${ANYLOG_TYPE}' — run 'bash deploy.sh help' for valid types" +} + +DOCKER_COMPOSE_FILE="docker-makefiles/docker-compose-files/${ANYLOG_TYPE}-docker-compose.yaml" + +# remove DOCKER_COMPOSE_FILE if exists +_cmd_clean_compose() { + if [[ "$IS_MANUAL" == "false" ]] && [[ -f ${DOCKER_COMPOSE_FILE} ]]; then + rm -rf "${DOCKER_COMPOSE_FILE}" + fi +} + + +# ────────────────────────────────────────────── +# Parse flags (--is-manual, --type, --tag, etc.) +# ────────────────────────────────────────────── +COMMAND="${1:-help}" +shift || true + +while [[ $# -gt 0 ]]; do + case "$1" in + --type|-t) ANYLOG_TYPE="$2"; shift 2 ;; + --tag) TAG="$2"; shift 2 ;; + --image) IMAGE="$2"; shift 2 ;; + --node-name) NODE_NAME="$2"; shift 2 ;; + --test-conn) TEST_CONN="$2"; shift 2 ;; + --license-key) export LICENSE_KEY="$2"; shift 2 ;; + --manual) IS_MANUAL="true"; shift ;; + --no-manual) IS_MANUAL="false"; shift ;; + *) die "Unknown option: $1" ;; + esac +done + +# Apply alias resolution and common setup +_resolve_type +_detect_runtime +_detect_platform +DOCKER_COMPOSE_FILE="docker-makefiles/docker-compose-files/${ANYLOG_TYPE}-docker-compose.yaml" + +# ────────────────────────────────────────────── +# Commands +# ────────────────────────────────────────────── + +cmd_login() { + echo "Logging into Docker Hub..." + ${CONTAINER_CMD} login docker.io -u anyloguser --password-stdin +} + +cmd_pull() { + _check_configs + _load_configs + echo "Pulling ${IMAGE}:${TAG}..." + ${CONTAINER_CMD} pull "docker.io/${IMAGE}:${TAG}" +} + +cmd_dry_run() { + _check_configs + _load_configs + if [[ "$IS_MANUAL" == "false" ]]; then + echo "Dry Run ${ANYLOG_TYPE} - ${NODE_NAME}" + bash docker-makefiles/prep_configs.sh "${ANYLOG_TYPE}" + bash docker-makefiles/build_docker_compose.sh "${ANYLOG_TYPE}" "${TAG}" + elif [[ "${IS_MANUAL}" == "true" ]]; then + local single_file="docker-makefiles/${ANYLOG_TYPE}/node_configs.env" + local deployments_repo + deployments_repo=$(grep -m1 '^DEPLOYMENTS_REPO=' "$single_file" 2>/dev/null | cut -d= -f2- | tr -d '"\r' || true) + + local vol_scripts + if [[ -n "$deployments_repo" && -d "$deployments_repo" ]]; then + vol_scripts="-v ${deployments_repo}:/app/deployment-scripts" + else + vol_scripts="-v ${NODE_NAME}-local-scripts:/app/deployment-scripts" + fi + + echo "Manual mode — docker run command:" + echo "" + echo " ${CONTAINER_CMD} run -it -d --detach-keys=ctrl-d \\" + echo " --name ${NODE_NAME} \\" + echo " --network host \\" + echo " --env-file ${single_file} \\" + echo " -v ${NODE_NAME}-anylog:/app/AnyLog-Network/anylog \\" + echo " -v ${NODE_NAME}-blockchain:/app/AnyLog-Network/blockchain \\" + echo " -v ${NODE_NAME}-data:/app/AnyLog-Network/data \\" + echo " ${vol_scripts} \\" + echo " --restart always \\" + echo " ${IMAGE}:${TAG}" + echo "" + else + echo "Dry Run skipped (manual mode)" + bash docker-makefiles/prep_configs.sh "${ANYLOG_TYPE}" + fi +} + +cmd_up() { + cmd_dry_run + if [[ "$IS_MANUAL" == "true" ]]; then + echo "Deploying ${ANYLOG_TYPE} [manual / docker run]" + local single_file="docker-makefiles/${ANYLOG_TYPE}/node_configs.env" + local deployments_repo + deployments_repo=$(grep -m1 '^DEPLOYMENTS_REPO=' "$single_file" 2>/dev/null | cut -d= -f2- | tr -d '"\r' || true) + + local vol_scripts + if [[ -n "$deployments_repo" && -d "$deployments_repo" ]]; then + vol_scripts="-v ${deployments_repo}:/app/deployment-scripts" + else + vol_scripts="-v ${NODE_NAME}-local-scripts:/app/deployment-scripts" + fi + + local license_flag="" + [[ -n "$LICENSE_KEY" ]] && license_flag="-e LICENSE_KEY=${LICENSE_KEY}" + ${CONTAINER_CMD} run -it -d --detach-keys=ctrl-d \ + --name "${NODE_NAME}" \ + --network host \ + --env-file "${single_file}" \ + ${license_flag} \ + -v "${NODE_NAME}-anylog:/app/AnyLog-Network/anylog" \ + -v "${NODE_NAME}-blockchain:/app/AnyLog-Network/blockchain" \ + -v "${NODE_NAME}-data:/app/AnyLog-Network/data" \ + ${vol_scripts} \ + --restart always \ + "${IMAGE}:${TAG}" + else + echo "Deploying ${ANYLOG_TYPE}" + LICENSE_KEY="${LICENSE_KEY}" ${DOCKER_COMPOSE_CMD} -f "${DOCKER_COMPOSE_FILE}" up -d + fi +} + +cmd_down() { + _check_configs + _load_configs + if [[ "$IS_MANUAL" == "true" ]]; then + echo "Stopping ${NODE_NAME} [manual]" + ${CONTAINER_CMD} stop "${NODE_NAME}" && ${CONTAINER_CMD} rm "${NODE_NAME}" + else + cmd_dry_run + echo "Stopping ${ANYLOG_TYPE}" + ${DOCKER_COMPOSE_CMD} -f "${DOCKER_COMPOSE_FILE}" down + fi +} + + +cmd_clean() { + _check_configs + _load_configs + if [[ "$IS_MANUAL" == "true" ]]; then + echo "Stopping + removing volumes: ${NODE_NAME} [manual]" + ${CONTAINER_CMD} stop "${NODE_NAME}" && ${CONTAINER_CMD} rm "${NODE_NAME}" + ${CONTAINER_CMD} volume rm \ + "${NODE_NAME}-anylog" \ + "${NODE_NAME}-blockchain" \ + "${NODE_NAME}-data" \ + "${NODE_NAME}-local-scripts" 2>/dev/null || true + else + cmd_dry_run + echo "Stopping + removing volumes: ${ANYLOG_TYPE}" + ${DOCKER_COMPOSE_CMD} -f "${DOCKER_COMPOSE_FILE}" down -v + fi + _cmd_clean_compose +} + +cmd_clean_all() { + _check_configs + _load_configs + if [[ "$IS_MANUAL" == "true" ]]; then + echo "Stopping + removing volumes + image: ${NODE_NAME} [manual]" + ${CONTAINER_CMD} stop "${NODE_NAME}" && ${CONTAINER_CMD} rm "${NODE_NAME}" + ${CONTAINER_CMD} volume rm \ + "${NODE_NAME}-anylog" \ + "${NODE_NAME}-blockchain" \ + "${NODE_NAME}-data" \ + "${NODE_NAME}-local-scripts" 2>/dev/null || true + ${CONTAINER_CMD} rmi "${IMAGE}:${TAG}" 2>/dev/null || true + else + cmd_dry_run + echo "Stopping + removing volumes + image: ${ANYLOG_TYPE}" + ${DOCKER_COMPOSE_CMD} -f "${DOCKER_COMPOSE_FILE}" down -v --rmi all + fi + _cmd_clean_compose +} + +cmd_logs() { + _check_configs; _load_configs + ${CONTAINER_CMD} logs "${NODE_NAME}" +} + +cmd_logs_f() { + _check_configs; _load_configs + ${CONTAINER_CMD} logs -f "${NODE_NAME}" +} + +cmd_attach() { + _check_configs; _load_configs + ${CONTAINER_CMD} attach --detach-keys=ctrl-d "${NODE_NAME}" +} + +cmd_exec() { + _check_configs; _load_configs + ${CONTAINER_CMD} exec -it "${NODE_NAME}" /bin/bash +} + +cmd_exec_root() { + _check_configs; _load_configs + ${CONTAINER_CMD} exec -u root -it "${NODE_NAME}" /bin/bash +} + +# ────────────────────────────────────────────── +# Testing +# ────────────────────────────────────────────── +_curl_anylog() { + local cmd="$1" + curl -s -X POST "http://${TEST_CONN}" \ + -H "Content-Type: application/json" \ + -d "{\"command\": \"${cmd}\", \"User-Agent\": \"AnyLog/1.23\"}" \ + -w "\n\n" +} + +cmd_test_status() { + _check_configs; _load_configs; _resolve_test_conn + echo "Check Status: ${TEST_CONN}" + _curl_anylog "get status where format=json" +} + +cmd_test_node() { + _check_configs; _load_configs; _resolve_test_conn + echo "Test Node: ${TEST_CONN}" + _curl_anylog "test node" +} + +cmd_test_network() { + _check_configs; _load_configs; _resolve_test_conn + echo "Test Network: ${TEST_CONN}" + _curl_anylog "test network" +} + +cmd_full_test() { + cmd_test_status + cmd_test_node + cmd_test_network +} + +cmd_check_processes() { + _check_configs; _load_configs; _resolve_test_conn + echo "Active/Inactive Services: ${TEST_CONN}" + _curl_anylog "get processes" +} + +# ────────────────────────────────────────────── +# check-vars / help +# ────────────────────────────────────────────── +cmd_check_vars() { + _load_configs 2>/dev/null || true + _resolve_test_conn 2>/dev/null || true + printf "%-22s %-30s %s\n" "Variable" "Default" "Value" + printf "%-22s %-30s %s\n" "--------" "-------" "-----" + printf "%-22s %-30s %s\n" "IS_MANUAL" "false" "$IS_MANUAL" + printf "%-22s %-30s %s\n" "ANYLOG_TYPE" "anylog-generic" "$ANYLOG_TYPE" + printf "%-22s %-30s %s\n" "IMAGE" "anylogco/anylog-network" "$IMAGE" + printf "%-22s %-30s %s\n" "NODE_NAME" "" "${NODE_NAME:-}" + printf "%-22s %-30s %s\n" "TAG" "pre-develop" "$TAG" + printf "%-22s %-30s %s\n" "LICENSE_KEY" "" "${LICENSE_KEY:-}" + printf "%-22s %-30s %s\n" "TEST_CONN" "127.0.0.1:" "$TEST_CONN" +} + +cmd_help() { + cat < [OPTIONS] + +Commands: + login Log into Docker Hub + pull Pull image from Docker Hub + dry-run Generate docker-compose.yaml (skipped in manual mode) + up Start AnyLog instance + down Stop AnyLog instance + clean Stop and remove volumes + clean-all Stop, remove volumes and image + logs View container logs + logs-f Follow container logs + attach Attach to container (ctrl-d to detach) + exec Shell into container (anylog user) + exec-root Shell into container (root) + full-test Run test-status + test-node + test-network + test-status GET status from node + test-node Test node configuration + test-network Test network connectivity + check-processes List active/inactive services + check-vars Show resolved variable values + help Show this message + +Options: + --type, -t Node type (generic, master, operator, query, publisher, + standalone-operator, standalone-publisher) + --tag Image tag (default: pre-develop) + --image Image repo (default: anylogco/anylog-network) + --node-name Override container name + --test-conn REST endpoint for test commands + --license-key License key to inject into container env + --manual Use docker run instead of docker compose + --no-manual Use docker compose (default) + +Examples: + bash deploy.sh up --type operator + bash deploy.sh up --type operator --manual + bash deploy.sh down --type anylog-operator + bash deploy.sh full-test --test-conn 192.168.1.10:32149 + bash deploy.sh exec-root --type master + bash deploy.sh check-vars --type query + +EOF +} + +# ────────────────────────────────────────────── +# Dispatch +# ────────────────────────────────────────────── +case "$COMMAND" in + login) cmd_login ;; + pull) cmd_pull ;; + dry-run) cmd_dry_run ;; + up) cmd_up ;; + down) cmd_down ;; + clean) cmd_clean ;; + clean-all) cmd_clean_all ;; + logs) cmd_logs ;; + logs-f) cmd_logs_f ;; + attach) cmd_attach ;; + exec) cmd_exec ;; + exec-root) cmd_exec_root ;; + full-test) cmd_full_test ;; + test-status) cmd_test_status ;; + test-node) cmd_test_node ;; + test-network) cmd_test_network ;; + check-processes) cmd_check_processes ;; + check-vars) cmd_check_vars ;; + help|--help|-h) cmd_help ;; + *) die "Unknown command '${COMMAND}' — run 'bash deploy.sh help'" ;; +esac \ No newline at end of file From fba24fd654d007bd65e7dbe9b62b3abb857dd7a6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 8 May 2026 23:48:25 +0000 Subject: [PATCH 16/46] chore: update version for os-dev [skip ci] --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index f9047bf8..1f23bab5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = docker-compose -version = 1.4.2604 (os-dev - e2d7bb [2026-05-07 20:31:29]) +version = 1.4.2604 (os-dev - 9be036 [2026-05-08 23:48:25]) author = AnyLog Co. contact = info@anylog.co description = docker-compose configurations for AnyLog deployment From c94ea462a6527b98001fe5723782dcb411806dfb Mon Sep 17 00:00:00 2001 From: Ori Shadmon Date: Tue, 12 May 2026 09:19:06 -0700 Subject: [PATCH 17/46] in progress - gRPC for customers + user-defined volumes Signed-off-by: Ori Shadmon --- Makefile | 161 +----------------- customers/dados.proto | 19 +++ customers/dados_pb2.py | 42 +++++ customers/dados_pb2_grpc.py | 97 +++++++++++ customers/grpc_client.al | 14 ++ deploy.sh | 2 +- .../node_configs.env | 4 +- docker-makefiles/build_docker_compose.sh | 2 +- docker-makefiles/prep_configs.sh | 2 +- 9 files changed, 186 insertions(+), 157 deletions(-) create mode 100644 customers/dados.proto create mode 100644 customers/dados_pb2.py create mode 100644 customers/dados_pb2_grpc.py create mode 100644 customers/grpc_client.al diff --git a/Makefile b/Makefile index 9dd4fe6b..c5c52c8d 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,6 @@ #!/bin/Makefile $(info LOADING MAKEFILE) -SHELL := /bin/bash - # ────────────────────────────────────────────── # Default values # ────────────────────────────────────────────── @@ -11,7 +9,6 @@ export ANYLOG_TYPE ?= anylog-generic export TAG ?= 1.4.2604 export IMAGE ?= anylogco/anylog-network export TEST_CONN ?= -export LICENSE_KEY ?= # Resolve short-form aliases (operator → anylog-operator) ifeq ($(ANYLOG_TYPE),$(filter $(ANYLOG_TYPE),generic master operator query publisher standalone-operator standalone-publisher)) @@ -33,8 +30,10 @@ ifneq ($(TEST_CONN),) _FLAGS += --test-conn $(TEST_CONN) endif -ANYLOG_SH := bash ./docker-makefiles/deploy.sh +ANYLOG_SH := bash deploy.sh +# ────────────────────────────────────────────── +all: help # ────────────────────────────────────────────── # Docker Hub @@ -42,7 +41,7 @@ ANYLOG_SH := bash ./docker-makefiles/deploy.sh login: ## log into Docker Hub for AnyLog $(ANYLOG_SH) login $(_FLAGS) -pull: .license_accepted ## pull image from Docker Hub +pull: ## pull image from Docker Hub $(ANYLOG_SH) pull $(_FLAGS) # ────────────────────────────────────────────── @@ -51,11 +50,8 @@ pull: .license_accepted ## pull image from Docker Hub dry-run: ## generate docker-compose.yaml (skipped in manual mode) $(ANYLOG_SH) dry-run $(_FLAGS) -# Read LICENSE_KEY from sentinel and export into the environment so deploy.sh -# and docker compose both inherit it — matching the same export pattern used -# by IS_MANUAL, ANYLOG_TYPE, TAG etc. at the top of this file. -up: .license_accepted syslog-setup ## start AnyLog instance (auto-configures rsyslog if SYSLOG_MONITORING=true) - @export LICENSE_KEY=$$(cut -d'|' -f5 .license_accepted); $(ANYLOG_SH) up $(_FLAGS) +up: ## start AnyLog instance + $(ANYLOG_SH) up $(_FLAGS) down: ## stop AnyLog instance $(ANYLOG_SH) down $(_FLAGS) @@ -84,138 +80,6 @@ exec: ## attach to bash shell (anylog user) exec-root: ## attach to bash shell as root $(ANYLOG_SH) exec-root $(_FLAGS) -# ────────────────────────────────────────────── -# Syslog forwarding -# ────────────────────────────────────────────── -# Reads SYSLOG_MONITORING and ANYLOG_BROKER_PORT from -# docker-makefiles/$(ANYLOG_TYPE)/node_configs.env — the same file deploy.sh -# passes as --env-file to docker run / docker compose. -# -# Linux (Ubuntu) — rsyslog drop-in /etc/rsyslog.d/60-custom-forwarding.conf (TCP) -# macOS (Darwin) — native syslogd via /etc/syslog.conf append (UDP) -# macOS syslogd only supports UDP forwarding; ensure -# ANYLOG_BROKER_PORT is open for UDP on the AnyLog node. -# -# Targets: -# syslog-setup — install the forwarding rule (idempotent) -# syslog-remove — remove the forwarding rule -# -# Both targets are no-ops when SYSLOG_MONITORING != true in node_configs.env. - -NODE_CONFIGS = docker-makefiles/$(ANYLOG_TYPE)/node_configs.env - -syslog-setup: ## configure syslog to forward to AnyLog msg port (reads node_configs.env) - @SYSLOG_MON=$$(grep -E '^SYSLOG_MONITORING=' $(NODE_CONFIGS) 2>/dev/null | head -1 | sed 's/^[^=]*=//;s/[[:space:]]*$$//'); \ - BROKER_PORT=$$(grep -E '^ANYLOG_BROKER_PORT=' $(NODE_CONFIGS) 2>/dev/null | head -1 | sed 's/^[^=]*=//;s/[[:space:]]*$$//'); \ - if [ "$$SYSLOG_MON" != "true" ]; then \ - echo "syslog-setup: SYSLOG_MONITORING is not 'true' in $(NODE_CONFIGS) — skipping."; \ - exit 0; \ - fi; \ - if [ -z "$$BROKER_PORT" ]; then \ - echo "ERROR: ANYLOG_BROKER_PORT is not set in $(NODE_CONFIGS)."; \ - exit 1; \ - fi; \ - \ - OS=$$(uname -s); \ - MARKER="# anylog-forwarding port=$$BROKER_PORT"; \ - \ - case "$$OS" in \ - \ - Linux) \ - if ! command -v rsyslogd >/dev/null 2>&1; then \ - echo "ERROR: rsyslogd not found. Install with: sudo apt-get install rsyslog"; \ - exit 1; \ - fi; \ - CONF_FILE="/etc/rsyslog.d/60-custom-forwarding.conf"; \ - if [ -f "$$CONF_FILE" ] && grep -qF "$$MARKER" "$$CONF_FILE" 2>/dev/null; then \ - echo "syslog-setup: rule already present in $$CONF_FILE — nothing to do."; \ - exit 0; \ - fi; \ - echo "Installing AnyLog rsyslog rule → $$CONF_FILE (TCP port $$BROKER_PORT)"; \ - printf '%s\n' \ - "$$MARKER" \ - "\$$template remote-incoming-logs, \"/var/log/remote/%%HOSTNAME%%.log\"" \ - "*.* ?remote-incoming-logs" \ - "*.* action(type=\"omfwd\" target=\"127.0.0.1\" port=\"$$BROKER_PORT\" protocol=\"tcp\")" \ - | sudo tee "$$CONF_FILE" > /dev/null; \ - sudo systemctl restart rsyslog \ - && echo "rsyslog restarted." \ - || echo "WARNING: could not restart rsyslog — run: sudo systemctl restart rsyslog"; \ - ;; \ - \ - Darwin) \ - CONF_FILE="/etc/syslog.conf"; \ - if [ ! -f "$$CONF_FILE" ]; then \ - echo "ERROR: $$CONF_FILE not found — is this macOS 10.12 or later?"; \ - echo " On macOS 12+ you may need to re-enable syslogd via launchctl."; \ - exit 1; \ - fi; \ - if grep -qF "$$MARKER" "$$CONF_FILE" 2>/dev/null; then \ - echo "syslog-setup: rule already present in $$CONF_FILE — nothing to do."; \ - exit 0; \ - fi; \ - echo "Installing AnyLog syslog rule → $$CONF_FILE (UDP port $$BROKER_PORT)"; \ - printf '\n%s\n%s\n' \ - "$$MARKER" \ - "*.* @127.0.0.1:$$BROKER_PORT" \ - | sudo tee -a "$$CONF_FILE" > /dev/null; \ - sudo launchctl kickstart -k system/com.apple.syslogd \ - && echo "syslogd restarted." \ - || echo "WARNING: could not restart syslogd — run: sudo launchctl kickstart -k system/com.apple.syslogd"; \ - ;; \ - \ - *) \ - echo "ERROR: Unsupported OS '$$OS'. Only Linux and macOS are supported."; \ - exit 1; \ - ;; \ - esac; \ - echo "syslog-setup: done." - -syslog-remove: ## remove the AnyLog syslog forwarding rule - @SYSLOG_MON=$$(grep -E '^SYSLOG_MONITORING=' $(NODE_CONFIGS) 2>/dev/null | head -1 | sed 's/^[^=]*=//;s/[[:space:]]*$$//'); \ - if [ "$$SYSLOG_MON" != "true" ]; then \ - echo "syslog-remove: SYSLOG_MONITORING is not 'true' in $(NODE_CONFIGS) — skipping."; \ - exit 0; \ - fi; \ - BROKER_PORT=$$(grep -E '^ANYLOG_BROKER_PORT=' $(NODE_CONFIGS) 2>/dev/null | head -1 | sed 's/^[^=]*=//;s/[[:space:]]*$$//'); \ - OS=$$(uname -s); \ - MARKER="# anylog-forwarding port=$$BROKER_PORT"; \ - \ - case "$$OS" in \ - \ - Linux) \ - CONF_FILE="/etc/rsyslog.d/60-custom-forwarding.conf"; \ - if [ ! -f "$$CONF_FILE" ]; then \ - echo "syslog-remove: $$CONF_FILE not found — nothing to remove."; \ - exit 0; \ - fi; \ - sudo rm -f "$$CONF_FILE"; \ - echo "Removed $$CONF_FILE"; \ - sudo systemctl restart rsyslog \ - && echo "rsyslog restarted." \ - || echo "WARNING: could not restart rsyslog — run: sudo systemctl restart rsyslog"; \ - ;; \ - \ - Darwin) \ - CONF_FILE="/etc/syslog.conf"; \ - if ! grep -qF "$$MARKER" "$$CONF_FILE" 2>/dev/null; then \ - echo "syslog-remove: AnyLog rule not found in $$CONF_FILE — nothing to remove."; \ - exit 0; \ - fi; \ - sudo sed -i '' "/^$$MARKER$$/{ N; d; }" "$$CONF_FILE"; \ - echo "Removed AnyLog forwarding rule from $$CONF_FILE"; \ - sudo launchctl kickstart -k system/com.apple.syslogd \ - && echo "syslogd restarted." \ - || echo "WARNING: could not restart syslogd — run: sudo launchctl kickstart -k system/com.apple.syslogd"; \ - ;; \ - \ - *) \ - echo "ERROR: Unsupported OS '$$OS'."; \ - exit 1; \ - ;; \ - esac; \ - echo "syslog-remove: done." - # ────────────────────────────────────────────── # Testing # ────────────────────────────────────────────── @@ -252,22 +116,15 @@ help: ## show this help message @echo " IS_MANUAL Use docker run instead of docker compose (default: false)" @echo " ANYLOG_TYPE Node type: generic master operator query publisher" @echo " standalone-operator standalone-publisher" - @echo " TAG Image tag (default: 1.4.2604)" + @echo " TAG Image tag (default: pre-develop)" @echo " IMAGE Image repository (default: anylogco/anylog-network)" @echo " TEST_CONN ip:port for test commands (default: auto-resolved)" - @echo " LICENSE_KEY License key (prompted on first run, stored in .license_accepted)" - @echo " LICENSE_URL License registration endpoint (default: http://127.0.0.1:8001/api/license-accept)" - @echo " SYSLOG_MONITORING Set to 'true' in node_configs.env to enable syslog → AnyLog forwarding" - @echo " Linux: rsyslog drop-in (TCP) | macOS: syslogd append (UDP)" - @echo " ANYLOG_BROKER_PORT Port syslog forwards to (read from node_configs.env)" @echo "" @echo "Without make:" @echo " bash deploy.sh help" @echo "" -.PHONY: license-check clean-license debug-key debug-post \ - login pull dry-run up down clean clean-all \ +.PHONY: login pull dry-run up down clean clean-all \ logs logs-f attach exec exec-root \ full-test test-status test-node test-network check-processes \ - check-vars help \ - syslog-setup syslog-remove + check-vars help \ No newline at end of file diff --git a/customers/dados.proto b/customers/dados.proto new file mode 100644 index 00000000..d05dbaf5 --- /dev/null +++ b/customers/dados.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; +package persistence; + +service PersistenceService { + rpc WatchTable (WatchRequest) returns (stream DataRow); +} + +message WatchRequest { + string table_id = 1; +} + +message DataRow { + string data = 1; +} + +// missing reply function +message ReplyMessage { + int32 Retval = 1; +} diff --git a/customers/dados_pb2.py b/customers/dados_pb2.py new file mode 100644 index 00000000..1f41d821 --- /dev/null +++ b/customers/dados_pb2.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: dados.proto +# Protobuf Python Version: 6.31.1 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 31, + 1, + '', + 'dados.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0b\x64\x61\x64os.proto\x12\x0bpersistence\" \n\x0cWatchRequest\x12\x10\n\x08table_id\x18\x01 \x01(\t\"\x17\n\x07\x44\x61taRow\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\t\"\x1e\n\x0cReplyMessage\x12\x0e\n\x06Retval\x18\x01 \x01(\x05\x32U\n\x12PersistenceService\x12?\n\nWatchTable\x12\x19.persistence.WatchRequest\x1a\x14.persistence.DataRow0\x01\x62\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'dados_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + DESCRIPTOR._loaded_options = None + _globals['_WATCHREQUEST']._serialized_start=28 + _globals['_WATCHREQUEST']._serialized_end=60 + _globals['_DATAROW']._serialized_start=62 + _globals['_DATAROW']._serialized_end=85 + _globals['_REPLYMESSAGE']._serialized_start=87 + _globals['_REPLYMESSAGE']._serialized_end=117 + _globals['_PERSISTENCESERVICE']._serialized_start=119 + _globals['_PERSISTENCESERVICE']._serialized_end=204 +# @@protoc_insertion_point(module_scope) diff --git a/customers/dados_pb2_grpc.py b/customers/dados_pb2_grpc.py new file mode 100644 index 00000000..d93c3bf0 --- /dev/null +++ b/customers/dados_pb2_grpc.py @@ -0,0 +1,97 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc +import warnings + +import dados_pb2 as dados__pb2 + +GRPC_GENERATED_VERSION = '1.76.0' +GRPC_VERSION = grpc.__version__ +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + raise RuntimeError( + f'The grpc package installed is at version {GRPC_VERSION},' + + ' but the generated code in dados_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + ) + + +class PersistenceServiceStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.WatchTable = channel.unary_stream( + '/persistence.PersistenceService/WatchTable', + request_serializer=dados__pb2.WatchRequest.SerializeToString, + response_deserializer=dados__pb2.DataRow.FromString, + _registered_method=True) + + +class PersistenceServiceServicer(object): + """Missing associated documentation comment in .proto file.""" + + def WatchTable(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_PersistenceServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'WatchTable': grpc.unary_stream_rpc_method_handler( + servicer.WatchTable, + request_deserializer=dados__pb2.WatchRequest.FromString, + response_serializer=dados__pb2.DataRow.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'persistence.PersistenceService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + server.add_registered_method_handlers('persistence.PersistenceService', rpc_method_handlers) + + + # This class is part of an EXPERIMENTAL API. +class PersistenceService(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def WatchTable(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream( + request, + target, + '/persistence.PersistenceService/WatchTable', + dados__pb2.WatchRequest.SerializeToString, + dados__pb2.DataRow.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/customers/grpc_client.al b/customers/grpc_client.al new file mode 100644 index 00000000..0b26a44b --- /dev/null +++ b/customers/grpc_client.al @@ -0,0 +1,14 @@ + \ No newline at end of file diff --git a/deploy.sh b/deploy.sh index 6c6607da..487c1756 100644 --- a/deploy.sh +++ b/deploy.sh @@ -2,7 +2,7 @@ # deploy.sh — AnyLog node lifecycle manager # Usage: bash deploy.sh [OPTIONS] # Run: bash deploy.sh help -#set -euo pipefail +set -euo pipefail # ────────────────────────────────────────────── # Defaults (override via environment or flags) diff --git a/docker-makefiles/anylog-standalone-operator/node_configs.env b/docker-makefiles/anylog-standalone-operator/node_configs.env index 3170d0dc..ebb760b7 100644 --- a/docker-makefiles/anylog-standalone-operator/node_configs.env +++ b/docker-makefiles/anylog-standalone-operator/node_configs.env @@ -38,7 +38,7 @@ DOCKER_SOCKET="/var/run/docker.sock" # Example: # /home/user/grpc-scripts -> /app/grpc-scripts # -USER_VOLUMES="" +USER_VOLUMES="/mnt/c/Users/oshad/AnyLog-code/docker-compose/customers" #==== basic ==== # Core node configuration - required for all deployments @@ -109,7 +109,7 @@ BLOCKCHAIN_DESTINATION=file # Owner of the cluster CLUSTER_NAME=anylog-cluster1 # Logical database name -DEFAULT_DBMS=new_company +DEFAULT_DBMS=persistence #==== southbound ==== # Data ingestion protocols (MQTT, OPC-UA, video streaming, monitoring) diff --git a/docker-makefiles/build_docker_compose.sh b/docker-makefiles/build_docker_compose.sh index b83be99c..34137453 100644 --- a/docker-makefiles/build_docker_compose.sh +++ b/docker-makefiles/build_docker_compose.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -#set -euo pipefail +set -euo pipefail # -------- Helpers -------- die() { diff --git a/docker-makefiles/prep_configs.sh b/docker-makefiles/prep_configs.sh index c3c4f30a..5d9a0d2c 100644 --- a/docker-makefiles/prep_configs.sh +++ b/docker-makefiles/prep_configs.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -#set -euo pipefail +set -euo pipefail # -------- Helpers -------- die() { From 44dc1495bff227e75684f8dcaafbbf2fcbb83389 Mon Sep 17 00:00:00 2001 From: Ori Shadmon Date: Tue, 12 May 2026 12:40:19 -0700 Subject: [PATCH 18/46] Makefile support for syslog Signed-off-by: Ori Shadmon --- support/Makefile | 27 ++++++ support/syslog.sh | 225 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 252 insertions(+) create mode 100644 support/syslog.sh diff --git a/support/Makefile b/support/Makefile index 23c71811..bf4e5b85 100644 --- a/support/Makefile +++ b/support/Makefile @@ -17,6 +17,9 @@ # make logs SERVICE=grafana # make exec SERVICE=postgres-prod # make list # show every discovered service +# make setup SERVICE=syslog # install syslog → AnyLog forwarding (prompts for config path) +# make setup SERVICE=syslog NODE_CONFIGS=../docker-makefiles/anylog-operator/node_configs.env +# make remove SERVICE=syslog NODE_CONFIGS=../docker-makefiles/anylog-operator/node_configs.env #---------------------------------------------------------------------- $(info LOADING MAKEFILE) @@ -50,6 +53,28 @@ endif all: help +# ── syslog intercept — resolved before any compose-based targets ─────── +# When SERVICE=syslog the public targets (setup/remove/help) delegate +# directly to syslog.sh instead of the docker-compose path. +# NODE_CONFIGS is prompted interactively when not supplied on the command line. +ifeq ($(_SVC),syslog) + +_syslog-config: + $(eval _SYSLOG_CFG := $(or $(NODE_CONFIGS),$(shell read -rp "Node config file path: " p && echo "$$p"))) + +setup: _syslog-config ## (syslog) Install host syslog → AnyLog forwarding rule + bash $(SUPPORT_DIR)syslog.sh setup "$(_SYSLOG_CFG)" + +remove: _syslog-config ## (syslog) Remove host syslog → AnyLog forwarding rule + bash $(SUPPORT_DIR)syslog.sh remove "$(_SYSLOG_CFG)" + +help: ## (syslog) Show syslog.sh usage + bash $(SUPPORT_DIR)syslog.sh help + +.PHONY: _syslog-config setup remove + +endif + # ====================================================================== # dry-run — (re)generate docker-compose.yml from configs.yaml # ====================================================================== @@ -222,6 +247,7 @@ help: ## Show this help message @echo " Default services : $(DEFAULT_SERVICES)" @echo " Aliases : gui -> remote-gui | psql -> postgres | mongo -> mongodb" @echo " Custom instances : create subdir + configs.yaml, then use SERVICE=" + @echo " Host syslog : SERVICE=syslog (delegates to syslog.sh, not docker-compose)" @echo "" @echo "Targets:" @grep -E '^[a-zA-Z_-]+:.*##' $(MAKEFILE_LIST) \ @@ -230,5 +256,6 @@ help: ## Show this help message .PHONY: all dry-run list help \ up down clean clean-all logs logs-f attach exec \ + setup remove \ _validate _ensure-compose \ _up _down _clean _clean-all _logs _logs-f _attach _exec \ No newline at end of file diff --git a/support/syslog.sh b/support/syslog.sh new file mode 100644 index 00000000..b0eeaa3c --- /dev/null +++ b/support/syslog.sh @@ -0,0 +1,225 @@ +#!/usr/bin/env bash +# syslog.sh — AnyLog syslog forwarding manager +# +# Configures the host syslog daemon to forward all log traffic to the AnyLog +# broker port. Reads SYSLOG_MONITORING and ANYLOG_BROKER_PORT from the node +# config file so this script stays in sync with the rest of the deployment +# without extra arguments. +# +# Usage: +# bash syslog.sh setup [NODE_CONFIGS] +# bash syslog.sh remove [NODE_CONFIGS] +# bash syslog.sh help +# +# Arguments: +# NODE_CONFIGS Path to node_configs.env (default: docker-makefiles/anylog-generic/node_configs.env) +# +# Platform behaviour: +# Linux — rsyslog drop-in /etc/rsyslog.d/60-custom-forwarding.conf (TCP) +# macOS — /etc/syslog.conf append (UDP) +# macOS syslogd only supports UDP; ensure ANYLOG_BROKER_PORT +# is open for UDP on the AnyLog node. +# +# Both commands are no-ops when SYSLOG_MONITORING != "true" in NODE_CONFIGS. +# setup is idempotent — safe to call repeatedly. + +set -euo pipefail + +# ────────────────────────────────────────────── +# Helpers +# ────────────────────────────────────────────── +die() { echo "ERROR: $1" >&2; exit "${2:-1}"; } + +# ────────────────────────────────────────────── +# Args +# ────────────────────────────────────────────── +COMMAND="${1:-help}" +NODE_CONFIGS="${2:-docker-makefiles/anylog-generic/node_configs.env}" + +# ────────────────────────────────────────────── +# Read variables from node config +# ────────────────────────────────────────────── +_load_syslog_vars() { + SYSLOG_MON=$(grep -E '^SYSLOG_MONITORING=' "${NODE_CONFIGS}" 2>/dev/null \ + | head -1 | sed 's/^[^=]*=//;s/[[:space:]]*$//' | tr -d '"'"'" ) + BROKER_PORT=$(grep -E '^ANYLOG_BROKER_PORT=' "${NODE_CONFIGS}" 2>/dev/null \ + | head -1 | sed 's/^[^=]*=//;s/[[:space:]]*$//' | tr -d '"'"'" ) +} + +# ────────────────────────────────────────────── +# setup +# ────────────────────────────────────────────── +cmd_setup() { + _load_syslog_vars + + if [[ "${SYSLOG_MON}" != "true" ]]; then + echo "syslog setup: SYSLOG_MONITORING is not 'true' in ${NODE_CONFIGS} — skipping." + exit 0 + fi + + [[ -n "${BROKER_PORT}" ]] || die "ANYLOG_BROKER_PORT is not set in ${NODE_CONFIGS}." + + local os marker + os=$(uname -s) + marker="# anylog-forwarding port=${BROKER_PORT}" + + case "${os}" in + + Linux) + command -v rsyslogd >/dev/null 2>&1 \ + || die "rsyslogd not found. Install with: sudo apt-get install rsyslog" + + local conf="/etc/rsyslog.d/60-custom-forwarding.conf" + + if [[ -f "${conf}" ]] && grep -qF "${marker}" "${conf}" 2>/dev/null; then + echo "syslog setup: rule already present in ${conf} — nothing to do." + exit 0 + fi + + echo "Installing AnyLog rsyslog rule → ${conf} (TCP port ${BROKER_PORT})" + printf '%s\n' \ + "${marker}" \ + "\$template remote-incoming-logs, \"/var/log/remote/%HOSTNAME%.log\"" \ + "*.* ?remote-incoming-logs" \ + "*.* action(type=\"omfwd\" target=\"127.0.0.1\" port=\"${BROKER_PORT}\" protocol=\"tcp\")" \ + | sudo tee "${conf}" > /dev/null + + sudo systemctl restart rsyslog \ + && echo "rsyslog restarted." \ + || echo "WARNING: could not restart rsyslog — run: sudo systemctl restart rsyslog" + ;; + + Darwin) + local conf="/etc/syslog.conf" + + [[ -f "${conf}" ]] \ + || die "${conf} not found — is this macOS 10.12 or later?\n On macOS 12+ you may need to re-enable syslogd via launchctl." + + if grep -qF "${marker}" "${conf}" 2>/dev/null; then + echo "syslog setup: rule already present in ${conf} — nothing to do." + exit 0 + fi + + echo "Installing AnyLog syslog rule → ${conf} (UDP port ${BROKER_PORT})" + printf '\n%s\n%s\n' \ + "${marker}" \ + "*.* @127.0.0.1:${BROKER_PORT}" \ + | sudo tee -a "${conf}" > /dev/null + + sudo launchctl kickstart -k system/com.apple.syslogd \ + && echo "syslogd restarted." \ + || echo "WARNING: could not restart syslogd — run: sudo launchctl kickstart -k system/com.apple.syslogd" + ;; + + *) + die "Unsupported OS '${os}'. Only Linux and macOS are supported." + ;; + esac + + echo "syslog setup: done." +} + +# ────────────────────────────────────────────── +# remove +# ────────────────────────────────────────────── +cmd_remove() { + _load_syslog_vars + + if [[ "${SYSLOG_MON}" != "true" ]]; then + echo "syslog remove: SYSLOG_MONITORING is not 'true' in ${NODE_CONFIGS} — skipping." + exit 0 + fi + + [[ -n "${BROKER_PORT}" ]] || die "ANYLOG_BROKER_PORT is not set in ${NODE_CONFIGS}." + + local os marker + os=$(uname -s) + marker="# anylog-forwarding port=${BROKER_PORT}" + + case "${os}" in + + Linux) + local conf="/etc/rsyslog.d/60-custom-forwarding.conf" + + if [[ ! -f "${conf}" ]]; then + echo "syslog remove: ${conf} not found — nothing to remove." + exit 0 + fi + + sudo rm -f "${conf}" + echo "Removed ${conf}" + + sudo systemctl restart rsyslog \ + && echo "rsyslog restarted." \ + || echo "WARNING: could not restart rsyslog — run: sudo systemctl restart rsyslog" + ;; + + Darwin) + local conf="/etc/syslog.conf" + + if ! grep -qF "${marker}" "${conf}" 2>/dev/null; then + echo "syslog remove: AnyLog rule not found in ${conf} — nothing to remove." + exit 0 + fi + + # Two-pass removal: delete the marker line and the forwarding rule that + # follows it. Using an address range is safer than { N; d; } which + # breaks when the marker falls on the last line of the file. + sudo sed -i '' "/${marker//\//\\/}/,/^\*\.\* @127\.0\.0\.1:${BROKER_PORT}$/d" "${conf}" + echo "Removed AnyLog forwarding rule from ${conf}" + + sudo launchctl kickstart -k system/com.apple.syslogd \ + && echo "syslogd restarted." \ + || echo "WARNING: could not restart syslogd — run: sudo launchctl kickstart -k system/com.apple.syslogd" + ;; + + *) + die "Unsupported OS '${os}'." + ;; + esac + + echo "syslog remove: done." +} + +# ────────────────────────────────────────────── +# help +# ────────────────────────────────────────────── +cmd_help() { + cat < [NODE_CONFIGS] + +Commands: + setup Configure host syslog to forward to AnyLog broker port (idempotent) + remove Remove the AnyLog syslog forwarding rule + help Show this message + +Arguments: + NODE_CONFIGS Path to node_configs.env + (default: docker-makefiles/anylog-generic/node_configs.env) + +Config keys read from NODE_CONFIGS: + SYSLOG_MONITORING Must be "true" to take any action + ANYLOG_BROKER_PORT Port syslog forwards to + +Platform behaviour: + Linux — rsyslog drop-in /etc/rsyslog.d/60-custom-forwarding.conf (TCP) + macOS — /etc/syslog.conf append (UDP) + +Examples: + bash syslog.sh setup + bash syslog.sh setup docker-makefiles/anylog-operator/node_configs.env + bash syslog.sh remove docker-makefiles/anylog-operator/node_configs.env + +EOF +} + +# ────────────────────────────────────────────── +# Dispatch +# ────────────────────────────────────────────── +case "${COMMAND}" in + setup) cmd_setup ;; + remove) cmd_remove ;; + help|--help|-h) cmd_help ;; + *) die "Unknown command '${COMMAND}' — run 'bash syslog.sh help'" ;; +esac \ No newline at end of file From 6b11d7b10af888dd04d8c2f3eb28b498fba0849e Mon Sep 17 00:00:00 2001 From: Ori Shadmon Date: Tue, 12 May 2026 12:40:40 -0700 Subject: [PATCH 19/46] in progress integreation of Mark's code into my pipeline Signed-off-by: Ori Shadmon --- Makefile | 161 +++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 152 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index c5c52c8d..9dd4fe6b 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,8 @@ #!/bin/Makefile $(info LOADING MAKEFILE) +SHELL := /bin/bash + # ────────────────────────────────────────────── # Default values # ────────────────────────────────────────────── @@ -9,6 +11,7 @@ export ANYLOG_TYPE ?= anylog-generic export TAG ?= 1.4.2604 export IMAGE ?= anylogco/anylog-network export TEST_CONN ?= +export LICENSE_KEY ?= # Resolve short-form aliases (operator → anylog-operator) ifeq ($(ANYLOG_TYPE),$(filter $(ANYLOG_TYPE),generic master operator query publisher standalone-operator standalone-publisher)) @@ -30,10 +33,8 @@ ifneq ($(TEST_CONN),) _FLAGS += --test-conn $(TEST_CONN) endif -ANYLOG_SH := bash deploy.sh +ANYLOG_SH := bash ./docker-makefiles/deploy.sh -# ────────────────────────────────────────────── -all: help # ────────────────────────────────────────────── # Docker Hub @@ -41,7 +42,7 @@ all: help login: ## log into Docker Hub for AnyLog $(ANYLOG_SH) login $(_FLAGS) -pull: ## pull image from Docker Hub +pull: .license_accepted ## pull image from Docker Hub $(ANYLOG_SH) pull $(_FLAGS) # ────────────────────────────────────────────── @@ -50,8 +51,11 @@ pull: ## pull image from Docker Hub dry-run: ## generate docker-compose.yaml (skipped in manual mode) $(ANYLOG_SH) dry-run $(_FLAGS) -up: ## start AnyLog instance - $(ANYLOG_SH) up $(_FLAGS) +# Read LICENSE_KEY from sentinel and export into the environment so deploy.sh +# and docker compose both inherit it — matching the same export pattern used +# by IS_MANUAL, ANYLOG_TYPE, TAG etc. at the top of this file. +up: .license_accepted syslog-setup ## start AnyLog instance (auto-configures rsyslog if SYSLOG_MONITORING=true) + @export LICENSE_KEY=$$(cut -d'|' -f5 .license_accepted); $(ANYLOG_SH) up $(_FLAGS) down: ## stop AnyLog instance $(ANYLOG_SH) down $(_FLAGS) @@ -80,6 +84,138 @@ exec: ## attach to bash shell (anylog user) exec-root: ## attach to bash shell as root $(ANYLOG_SH) exec-root $(_FLAGS) +# ────────────────────────────────────────────── +# Syslog forwarding +# ────────────────────────────────────────────── +# Reads SYSLOG_MONITORING and ANYLOG_BROKER_PORT from +# docker-makefiles/$(ANYLOG_TYPE)/node_configs.env — the same file deploy.sh +# passes as --env-file to docker run / docker compose. +# +# Linux (Ubuntu) — rsyslog drop-in /etc/rsyslog.d/60-custom-forwarding.conf (TCP) +# macOS (Darwin) — native syslogd via /etc/syslog.conf append (UDP) +# macOS syslogd only supports UDP forwarding; ensure +# ANYLOG_BROKER_PORT is open for UDP on the AnyLog node. +# +# Targets: +# syslog-setup — install the forwarding rule (idempotent) +# syslog-remove — remove the forwarding rule +# +# Both targets are no-ops when SYSLOG_MONITORING != true in node_configs.env. + +NODE_CONFIGS = docker-makefiles/$(ANYLOG_TYPE)/node_configs.env + +syslog-setup: ## configure syslog to forward to AnyLog msg port (reads node_configs.env) + @SYSLOG_MON=$$(grep -E '^SYSLOG_MONITORING=' $(NODE_CONFIGS) 2>/dev/null | head -1 | sed 's/^[^=]*=//;s/[[:space:]]*$$//'); \ + BROKER_PORT=$$(grep -E '^ANYLOG_BROKER_PORT=' $(NODE_CONFIGS) 2>/dev/null | head -1 | sed 's/^[^=]*=//;s/[[:space:]]*$$//'); \ + if [ "$$SYSLOG_MON" != "true" ]; then \ + echo "syslog-setup: SYSLOG_MONITORING is not 'true' in $(NODE_CONFIGS) — skipping."; \ + exit 0; \ + fi; \ + if [ -z "$$BROKER_PORT" ]; then \ + echo "ERROR: ANYLOG_BROKER_PORT is not set in $(NODE_CONFIGS)."; \ + exit 1; \ + fi; \ + \ + OS=$$(uname -s); \ + MARKER="# anylog-forwarding port=$$BROKER_PORT"; \ + \ + case "$$OS" in \ + \ + Linux) \ + if ! command -v rsyslogd >/dev/null 2>&1; then \ + echo "ERROR: rsyslogd not found. Install with: sudo apt-get install rsyslog"; \ + exit 1; \ + fi; \ + CONF_FILE="/etc/rsyslog.d/60-custom-forwarding.conf"; \ + if [ -f "$$CONF_FILE" ] && grep -qF "$$MARKER" "$$CONF_FILE" 2>/dev/null; then \ + echo "syslog-setup: rule already present in $$CONF_FILE — nothing to do."; \ + exit 0; \ + fi; \ + echo "Installing AnyLog rsyslog rule → $$CONF_FILE (TCP port $$BROKER_PORT)"; \ + printf '%s\n' \ + "$$MARKER" \ + "\$$template remote-incoming-logs, \"/var/log/remote/%%HOSTNAME%%.log\"" \ + "*.* ?remote-incoming-logs" \ + "*.* action(type=\"omfwd\" target=\"127.0.0.1\" port=\"$$BROKER_PORT\" protocol=\"tcp\")" \ + | sudo tee "$$CONF_FILE" > /dev/null; \ + sudo systemctl restart rsyslog \ + && echo "rsyslog restarted." \ + || echo "WARNING: could not restart rsyslog — run: sudo systemctl restart rsyslog"; \ + ;; \ + \ + Darwin) \ + CONF_FILE="/etc/syslog.conf"; \ + if [ ! -f "$$CONF_FILE" ]; then \ + echo "ERROR: $$CONF_FILE not found — is this macOS 10.12 or later?"; \ + echo " On macOS 12+ you may need to re-enable syslogd via launchctl."; \ + exit 1; \ + fi; \ + if grep -qF "$$MARKER" "$$CONF_FILE" 2>/dev/null; then \ + echo "syslog-setup: rule already present in $$CONF_FILE — nothing to do."; \ + exit 0; \ + fi; \ + echo "Installing AnyLog syslog rule → $$CONF_FILE (UDP port $$BROKER_PORT)"; \ + printf '\n%s\n%s\n' \ + "$$MARKER" \ + "*.* @127.0.0.1:$$BROKER_PORT" \ + | sudo tee -a "$$CONF_FILE" > /dev/null; \ + sudo launchctl kickstart -k system/com.apple.syslogd \ + && echo "syslogd restarted." \ + || echo "WARNING: could not restart syslogd — run: sudo launchctl kickstart -k system/com.apple.syslogd"; \ + ;; \ + \ + *) \ + echo "ERROR: Unsupported OS '$$OS'. Only Linux and macOS are supported."; \ + exit 1; \ + ;; \ + esac; \ + echo "syslog-setup: done." + +syslog-remove: ## remove the AnyLog syslog forwarding rule + @SYSLOG_MON=$$(grep -E '^SYSLOG_MONITORING=' $(NODE_CONFIGS) 2>/dev/null | head -1 | sed 's/^[^=]*=//;s/[[:space:]]*$$//'); \ + if [ "$$SYSLOG_MON" != "true" ]; then \ + echo "syslog-remove: SYSLOG_MONITORING is not 'true' in $(NODE_CONFIGS) — skipping."; \ + exit 0; \ + fi; \ + BROKER_PORT=$$(grep -E '^ANYLOG_BROKER_PORT=' $(NODE_CONFIGS) 2>/dev/null | head -1 | sed 's/^[^=]*=//;s/[[:space:]]*$$//'); \ + OS=$$(uname -s); \ + MARKER="# anylog-forwarding port=$$BROKER_PORT"; \ + \ + case "$$OS" in \ + \ + Linux) \ + CONF_FILE="/etc/rsyslog.d/60-custom-forwarding.conf"; \ + if [ ! -f "$$CONF_FILE" ]; then \ + echo "syslog-remove: $$CONF_FILE not found — nothing to remove."; \ + exit 0; \ + fi; \ + sudo rm -f "$$CONF_FILE"; \ + echo "Removed $$CONF_FILE"; \ + sudo systemctl restart rsyslog \ + && echo "rsyslog restarted." \ + || echo "WARNING: could not restart rsyslog — run: sudo systemctl restart rsyslog"; \ + ;; \ + \ + Darwin) \ + CONF_FILE="/etc/syslog.conf"; \ + if ! grep -qF "$$MARKER" "$$CONF_FILE" 2>/dev/null; then \ + echo "syslog-remove: AnyLog rule not found in $$CONF_FILE — nothing to remove."; \ + exit 0; \ + fi; \ + sudo sed -i '' "/^$$MARKER$$/{ N; d; }" "$$CONF_FILE"; \ + echo "Removed AnyLog forwarding rule from $$CONF_FILE"; \ + sudo launchctl kickstart -k system/com.apple.syslogd \ + && echo "syslogd restarted." \ + || echo "WARNING: could not restart syslogd — run: sudo launchctl kickstart -k system/com.apple.syslogd"; \ + ;; \ + \ + *) \ + echo "ERROR: Unsupported OS '$$OS'."; \ + exit 1; \ + ;; \ + esac; \ + echo "syslog-remove: done." + # ────────────────────────────────────────────── # Testing # ────────────────────────────────────────────── @@ -116,15 +252,22 @@ help: ## show this help message @echo " IS_MANUAL Use docker run instead of docker compose (default: false)" @echo " ANYLOG_TYPE Node type: generic master operator query publisher" @echo " standalone-operator standalone-publisher" - @echo " TAG Image tag (default: pre-develop)" + @echo " TAG Image tag (default: 1.4.2604)" @echo " IMAGE Image repository (default: anylogco/anylog-network)" @echo " TEST_CONN ip:port for test commands (default: auto-resolved)" + @echo " LICENSE_KEY License key (prompted on first run, stored in .license_accepted)" + @echo " LICENSE_URL License registration endpoint (default: http://127.0.0.1:8001/api/license-accept)" + @echo " SYSLOG_MONITORING Set to 'true' in node_configs.env to enable syslog → AnyLog forwarding" + @echo " Linux: rsyslog drop-in (TCP) | macOS: syslogd append (UDP)" + @echo " ANYLOG_BROKER_PORT Port syslog forwards to (read from node_configs.env)" @echo "" @echo "Without make:" @echo " bash deploy.sh help" @echo "" -.PHONY: login pull dry-run up down clean clean-all \ +.PHONY: license-check clean-license debug-key debug-post \ + login pull dry-run up down clean clean-all \ logs logs-f attach exec exec-root \ full-test test-status test-node test-network check-processes \ - check-vars help \ No newline at end of file + check-vars help \ + syslog-setup syslog-remove From ba48cd033b438494b193957cce41f17c367d0ddc Mon Sep 17 00:00:00 2001 From: Ori Shadmon Date: Tue, 12 May 2026 12:47:06 -0700 Subject: [PATCH 20/46] merge license key info to a single file Signed-off-by: Ori Shadmon --- {support => license-key-generator}/README.license_server.md | 0 license-key-generator/__init__.py | 0 .../docker-compose.license_server.yml | 4 ++-- {support => license-key-generator}/license-server.py | 0 .../requirements.license_server.txt | 0 5 files changed, 2 insertions(+), 2 deletions(-) rename {support => license-key-generator}/README.license_server.md (100%) create mode 100644 license-key-generator/__init__.py rename {support => license-key-generator}/docker-compose.license_server.yml (73%) rename {support => license-key-generator}/license-server.py (100%) rename {support => license-key-generator}/requirements.license_server.txt (100%) diff --git a/support/README.license_server.md b/license-key-generator/README.license_server.md similarity index 100% rename from support/README.license_server.md rename to license-key-generator/README.license_server.md diff --git a/license-key-generator/__init__.py b/license-key-generator/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/support/docker-compose.license_server.yml b/license-key-generator/docker-compose.license_server.yml similarity index 73% rename from support/docker-compose.license_server.yml rename to license-key-generator/docker-compose.license_server.yml index 18d5aee1..16a3707d 100644 --- a/support/docker-compose.license_server.yml +++ b/license-key-generator/docker-compose.license_server.yml @@ -1,8 +1,8 @@ services: license-server: build: - context: . - dockerfile: Dockerfile.license_server + context: ../support + dockerfile: ../support/Dockerfile.license_server container_name: license-server restart: unless-stopped ports: diff --git a/support/license-server.py b/license-key-generator/license-server.py similarity index 100% rename from support/license-server.py rename to license-key-generator/license-server.py diff --git a/support/requirements.license_server.txt b/license-key-generator/requirements.license_server.txt similarity index 100% rename from support/requirements.license_server.txt rename to license-key-generator/requirements.license_server.txt From 7eefa646b913e224a7c20e6176609cd7f8d376b8 Mon Sep 17 00:00:00 2001 From: Ori Shadmon Date: Tue, 12 May 2026 13:06:32 -0700 Subject: [PATCH 21/46] license generator in single location Signed-off-by: Ori Shadmon --- .../Dockerfile | 6 +++--- .../README.md | 10 +++++----- {license-key-generator => license-generator}/deploy.sh | 0 .../docker-compose.yml | 4 ++-- .../requirements.txt | 0 .../src}/__init__.py | 0 .../src}/license-server.py | 0 7 files changed, 10 insertions(+), 10 deletions(-) rename support/Dockerfile.license_server => license-generator/Dockerfile (58%) rename license-key-generator/README.license_server.md => license-generator/README.md (82%) rename {license-key-generator => license-generator}/deploy.sh (100%) rename license-key-generator/docker-compose.license_server.yml => license-generator/docker-compose.yml (73%) rename license-key-generator/requirements.license_server.txt => license-generator/requirements.txt (100%) rename {license-key-generator => license-generator/src}/__init__.py (100%) rename {license-key-generator => license-generator/src}/license-server.py (100%) diff --git a/support/Dockerfile.license_server b/license-generator/Dockerfile similarity index 58% rename from support/Dockerfile.license_server rename to license-generator/Dockerfile index c7324b7e..5a33e20d 100644 --- a/support/Dockerfile.license_server +++ b/license-generator/Dockerfile @@ -2,10 +2,10 @@ FROM python:3.13-slim WORKDIR /app -COPY requirements.license_server.txt . -RUN pip install --no-cache-dir -r requirements.license_server.txt +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt -COPY license-server.py app.py +COPY src/license-server.py app.py RUN mkdir -p /data diff --git a/license-key-generator/README.license_server.md b/license-generator/README.md similarity index 82% rename from license-key-generator/README.license_server.md rename to license-generator/README.md index b54d595b..f2e28385 100644 --- a/license-key-generator/README.license_server.md +++ b/license-generator/README.md @@ -20,19 +20,19 @@ The Dockerfile copies `license-server.py` into the container as `app.py` so Guni Run from the directory containing `docker-compose.license_server.yml`: ```bash -docker compose -f docker-compose.license_server.yml up -d --build +docker compose -f docker-compose.yml up -d --build ``` ## Check Container Status ```bash -docker compose -f docker-compose.license_server.yml ps +docker compose -f docker-compose.yml ps ``` ## View Logs ```bash -docker compose -f docker-compose.license_server.yml logs -f +docker compose -f docker-compose.yml logs -f ``` ## Health Check @@ -87,11 +87,11 @@ This file remains available even if the container is restarted or rebuilt. ## Stop the Service ```bash -docker compose -f docker-compose.license_server.yml down +docker compose -f docker-compose.yml down ``` ## Rebuild After Code Changes ```bash -docker compose -f docker-compose.license_server.yml up -d --build +docker compose -f docker-compose.yml up -d --build ``` diff --git a/license-key-generator/deploy.sh b/license-generator/deploy.sh similarity index 100% rename from license-key-generator/deploy.sh rename to license-generator/deploy.sh diff --git a/license-key-generator/docker-compose.license_server.yml b/license-generator/docker-compose.yml similarity index 73% rename from license-key-generator/docker-compose.license_server.yml rename to license-generator/docker-compose.yml index 16a3707d..aa1390a6 100644 --- a/license-key-generator/docker-compose.license_server.yml +++ b/license-generator/docker-compose.yml @@ -1,8 +1,8 @@ services: license-server: build: - context: ../support - dockerfile: ../support/Dockerfile.license_server + context: . + dockerfile: Dockerfile container_name: license-server restart: unless-stopped ports: diff --git a/license-key-generator/requirements.license_server.txt b/license-generator/requirements.txt similarity index 100% rename from license-key-generator/requirements.license_server.txt rename to license-generator/requirements.txt diff --git a/license-key-generator/__init__.py b/license-generator/src/__init__.py similarity index 100% rename from license-key-generator/__init__.py rename to license-generator/src/__init__.py diff --git a/license-key-generator/license-server.py b/license-generator/src/license-server.py similarity index 100% rename from license-key-generator/license-server.py rename to license-generator/src/license-server.py From 1c8f101e8b10a17ee198d0a2c9d1d5c6aafc6625 Mon Sep 17 00:00:00 2001 From: Ori Shadmon Date: Tue, 12 May 2026 14:13:37 -0700 Subject: [PATCH 22/46] in progress Signed-off-by: Ori Shadmon --- Makefile | 142 ++----------------------------- deploy.sh | 18 ++-- license-generator/license_key.sh | 76 +++++++++++++++++ 3 files changed, 92 insertions(+), 144 deletions(-) create mode 100644 license-generator/license_key.sh diff --git a/Makefile b/Makefile index 9dd4fe6b..4b7f3990 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ ifneq ($(TEST_CONN),) _FLAGS += --test-conn $(TEST_CONN) endif -ANYLOG_SH := bash ./docker-makefiles/deploy.sh +ANYLOG_SH := bash -x ./docker-makefiles/deploy.sh # ────────────────────────────────────────────── @@ -42,7 +42,7 @@ ANYLOG_SH := bash ./docker-makefiles/deploy.sh login: ## log into Docker Hub for AnyLog $(ANYLOG_SH) login $(_FLAGS) -pull: .license_accepted ## pull image from Docker Hub +pull: ## pull image from Docker Hub $(ANYLOG_SH) pull $(_FLAGS) # ────────────────────────────────────────────── @@ -54,8 +54,8 @@ dry-run: ## generate docker-compose.yaml (skipped in manual mode) # Read LICENSE_KEY from sentinel and export into the environment so deploy.sh # and docker compose both inherit it — matching the same export pattern used # by IS_MANUAL, ANYLOG_TYPE, TAG etc. at the top of this file. -up: .license_accepted syslog-setup ## start AnyLog instance (auto-configures rsyslog if SYSLOG_MONITORING=true) - @export LICENSE_KEY=$$(cut -d'|' -f5 .license_accepted); $(ANYLOG_SH) up $(_FLAGS) +up: ## start AnyLog instance (auto-configures rsyslog if SYSLOG_MONITORING=true) + @export LICENSE_KEY=$$(cut -d'|' -f5); $(ANYLOG_SH) up $(_FLAGS) down: ## stop AnyLog instance $(ANYLOG_SH) down $(_FLAGS) @@ -84,138 +84,6 @@ exec: ## attach to bash shell (anylog user) exec-root: ## attach to bash shell as root $(ANYLOG_SH) exec-root $(_FLAGS) -# ────────────────────────────────────────────── -# Syslog forwarding -# ────────────────────────────────────────────── -# Reads SYSLOG_MONITORING and ANYLOG_BROKER_PORT from -# docker-makefiles/$(ANYLOG_TYPE)/node_configs.env — the same file deploy.sh -# passes as --env-file to docker run / docker compose. -# -# Linux (Ubuntu) — rsyslog drop-in /etc/rsyslog.d/60-custom-forwarding.conf (TCP) -# macOS (Darwin) — native syslogd via /etc/syslog.conf append (UDP) -# macOS syslogd only supports UDP forwarding; ensure -# ANYLOG_BROKER_PORT is open for UDP on the AnyLog node. -# -# Targets: -# syslog-setup — install the forwarding rule (idempotent) -# syslog-remove — remove the forwarding rule -# -# Both targets are no-ops when SYSLOG_MONITORING != true in node_configs.env. - -NODE_CONFIGS = docker-makefiles/$(ANYLOG_TYPE)/node_configs.env - -syslog-setup: ## configure syslog to forward to AnyLog msg port (reads node_configs.env) - @SYSLOG_MON=$$(grep -E '^SYSLOG_MONITORING=' $(NODE_CONFIGS) 2>/dev/null | head -1 | sed 's/^[^=]*=//;s/[[:space:]]*$$//'); \ - BROKER_PORT=$$(grep -E '^ANYLOG_BROKER_PORT=' $(NODE_CONFIGS) 2>/dev/null | head -1 | sed 's/^[^=]*=//;s/[[:space:]]*$$//'); \ - if [ "$$SYSLOG_MON" != "true" ]; then \ - echo "syslog-setup: SYSLOG_MONITORING is not 'true' in $(NODE_CONFIGS) — skipping."; \ - exit 0; \ - fi; \ - if [ -z "$$BROKER_PORT" ]; then \ - echo "ERROR: ANYLOG_BROKER_PORT is not set in $(NODE_CONFIGS)."; \ - exit 1; \ - fi; \ - \ - OS=$$(uname -s); \ - MARKER="# anylog-forwarding port=$$BROKER_PORT"; \ - \ - case "$$OS" in \ - \ - Linux) \ - if ! command -v rsyslogd >/dev/null 2>&1; then \ - echo "ERROR: rsyslogd not found. Install with: sudo apt-get install rsyslog"; \ - exit 1; \ - fi; \ - CONF_FILE="/etc/rsyslog.d/60-custom-forwarding.conf"; \ - if [ -f "$$CONF_FILE" ] && grep -qF "$$MARKER" "$$CONF_FILE" 2>/dev/null; then \ - echo "syslog-setup: rule already present in $$CONF_FILE — nothing to do."; \ - exit 0; \ - fi; \ - echo "Installing AnyLog rsyslog rule → $$CONF_FILE (TCP port $$BROKER_PORT)"; \ - printf '%s\n' \ - "$$MARKER" \ - "\$$template remote-incoming-logs, \"/var/log/remote/%%HOSTNAME%%.log\"" \ - "*.* ?remote-incoming-logs" \ - "*.* action(type=\"omfwd\" target=\"127.0.0.1\" port=\"$$BROKER_PORT\" protocol=\"tcp\")" \ - | sudo tee "$$CONF_FILE" > /dev/null; \ - sudo systemctl restart rsyslog \ - && echo "rsyslog restarted." \ - || echo "WARNING: could not restart rsyslog — run: sudo systemctl restart rsyslog"; \ - ;; \ - \ - Darwin) \ - CONF_FILE="/etc/syslog.conf"; \ - if [ ! -f "$$CONF_FILE" ]; then \ - echo "ERROR: $$CONF_FILE not found — is this macOS 10.12 or later?"; \ - echo " On macOS 12+ you may need to re-enable syslogd via launchctl."; \ - exit 1; \ - fi; \ - if grep -qF "$$MARKER" "$$CONF_FILE" 2>/dev/null; then \ - echo "syslog-setup: rule already present in $$CONF_FILE — nothing to do."; \ - exit 0; \ - fi; \ - echo "Installing AnyLog syslog rule → $$CONF_FILE (UDP port $$BROKER_PORT)"; \ - printf '\n%s\n%s\n' \ - "$$MARKER" \ - "*.* @127.0.0.1:$$BROKER_PORT" \ - | sudo tee -a "$$CONF_FILE" > /dev/null; \ - sudo launchctl kickstart -k system/com.apple.syslogd \ - && echo "syslogd restarted." \ - || echo "WARNING: could not restart syslogd — run: sudo launchctl kickstart -k system/com.apple.syslogd"; \ - ;; \ - \ - *) \ - echo "ERROR: Unsupported OS '$$OS'. Only Linux and macOS are supported."; \ - exit 1; \ - ;; \ - esac; \ - echo "syslog-setup: done." - -syslog-remove: ## remove the AnyLog syslog forwarding rule - @SYSLOG_MON=$$(grep -E '^SYSLOG_MONITORING=' $(NODE_CONFIGS) 2>/dev/null | head -1 | sed 's/^[^=]*=//;s/[[:space:]]*$$//'); \ - if [ "$$SYSLOG_MON" != "true" ]; then \ - echo "syslog-remove: SYSLOG_MONITORING is not 'true' in $(NODE_CONFIGS) — skipping."; \ - exit 0; \ - fi; \ - BROKER_PORT=$$(grep -E '^ANYLOG_BROKER_PORT=' $(NODE_CONFIGS) 2>/dev/null | head -1 | sed 's/^[^=]*=//;s/[[:space:]]*$$//'); \ - OS=$$(uname -s); \ - MARKER="# anylog-forwarding port=$$BROKER_PORT"; \ - \ - case "$$OS" in \ - \ - Linux) \ - CONF_FILE="/etc/rsyslog.d/60-custom-forwarding.conf"; \ - if [ ! -f "$$CONF_FILE" ]; then \ - echo "syslog-remove: $$CONF_FILE not found — nothing to remove."; \ - exit 0; \ - fi; \ - sudo rm -f "$$CONF_FILE"; \ - echo "Removed $$CONF_FILE"; \ - sudo systemctl restart rsyslog \ - && echo "rsyslog restarted." \ - || echo "WARNING: could not restart rsyslog — run: sudo systemctl restart rsyslog"; \ - ;; \ - \ - Darwin) \ - CONF_FILE="/etc/syslog.conf"; \ - if ! grep -qF "$$MARKER" "$$CONF_FILE" 2>/dev/null; then \ - echo "syslog-remove: AnyLog rule not found in $$CONF_FILE — nothing to remove."; \ - exit 0; \ - fi; \ - sudo sed -i '' "/^$$MARKER$$/{ N; d; }" "$$CONF_FILE"; \ - echo "Removed AnyLog forwarding rule from $$CONF_FILE"; \ - sudo launchctl kickstart -k system/com.apple.syslogd \ - && echo "syslogd restarted." \ - || echo "WARNING: could not restart syslogd — run: sudo launchctl kickstart -k system/com.apple.syslogd"; \ - ;; \ - \ - *) \ - echo "ERROR: Unsupported OS '$$OS'."; \ - exit 1; \ - ;; \ - esac; \ - echo "syslog-remove: done." - # ────────────────────────────────────────────── # Testing # ────────────────────────────────────────────── @@ -255,7 +123,7 @@ help: ## show this help message @echo " TAG Image tag (default: 1.4.2604)" @echo " IMAGE Image repository (default: anylogco/anylog-network)" @echo " TEST_CONN ip:port for test commands (default: auto-resolved)" - @echo " LICENSE_KEY License key (prompted on first run, stored in .license_accepted)" + @echo " LICENSE_KEY License key (prompted on first run, stored in)" @echo " LICENSE_URL License registration endpoint (default: http://127.0.0.1:8001/api/license-accept)" @echo " SYSLOG_MONITORING Set to 'true' in node_configs.env to enable syslog → AnyLog forwarding" @echo " Linux: rsyslog drop-in (TCP) | macOS: syslogd append (UDP)" diff --git a/deploy.sh b/deploy.sh index 487c1756..af1d43c3 100644 --- a/deploy.sh +++ b/deploy.sh @@ -7,12 +7,7 @@ set -euo pipefail # ────────────────────────────────────────────── # Defaults (override via environment or flags) # ────────────────────────────────────────────── -IS_MANUAL="${IS_MANUAL:-false}" -ANYLOG_TYPE="${ANYLOG_TYPE:-anylog-generic}" -TAG="${TAG:-pre-develop}" -IMAGE="${IMAGE:-anylogco/anylog-network}" -TEST_CONN="${TEST_CONN:-}" -NODE_NAME="${NODE_NAME:-}" +export LICENSE_KEY="${LICENSE_KEY:-}" # ────────────────────────────────────────────── # Helpers @@ -113,6 +108,7 @@ while [[ $# -gt 0 ]]; do --image) IMAGE="$2"; shift 2 ;; --node-name) NODE_NAME="$2"; shift 2 ;; --test-conn) TEST_CONN="$2"; shift 2 ;; + --license-key) export LICENSE_KEY="$2"; shift 2 ;; --manual) IS_MANUAL="true"; shift ;; --no-manual) IS_MANUAL="false"; shift ;; *) die "Unknown option: $1" ;; @@ -137,6 +133,7 @@ cmd_login() { cmd_pull() { _check_configs _load_configs + bash license-generator/license_key.sh "${ANYLOG_TYPE}" echo "Pulling ${IMAGE}:${TAG}..." ${CONTAINER_CMD} pull "docker.io/${IMAGE}:${TAG}" } @@ -181,6 +178,8 @@ cmd_dry_run() { cmd_up() { cmd_dry_run + echo "LICENSE_KEY: ${ANYLOG_TYPE}" + bash docker-makefiles/license.sh "${ANYLOG_TYPE}" if [[ "$IS_MANUAL" == "true" ]]; then echo "Deploying ${ANYLOG_TYPE} [manual / docker run]" local single_file="docker-makefiles/${ANYLOG_TYPE}/node_configs.env" @@ -194,10 +193,13 @@ cmd_up() { vol_scripts="-v ${NODE_NAME}-local-scripts:/app/deployment-scripts" fi + local license_flag="" + [[ -n "$LICENSE_KEY" ]] && license_flag="-e LICENSE_KEY=${LICENSE_KEY}" ${CONTAINER_CMD} run -it -d --detach-keys=ctrl-d \ --name "${NODE_NAME}" \ --network host \ --env-file "${single_file}" \ + ${license_flag} \ -v "${NODE_NAME}-anylog:/app/AnyLog-Network/anylog" \ -v "${NODE_NAME}-blockchain:/app/AnyLog-Network/blockchain" \ -v "${NODE_NAME}-data:/app/AnyLog-Network/data" \ @@ -206,7 +208,7 @@ cmd_up() { "${IMAGE}:${TAG}" else echo "Deploying ${ANYLOG_TYPE}" - ${DOCKER_COMPOSE_CMD} -f "${DOCKER_COMPOSE_FILE}" up -d + LICENSE_KEY="${LICENSE_KEY}" ${DOCKER_COMPOSE_CMD} -f "${DOCKER_COMPOSE_FILE}" up -d fi } @@ -342,6 +344,7 @@ cmd_check_vars() { printf "%-22s %-30s %s\n" "IMAGE" "anylogco/anylog-network" "$IMAGE" printf "%-22s %-30s %s\n" "NODE_NAME" "" "${NODE_NAME:-}" printf "%-22s %-30s %s\n" "TAG" "pre-develop" "$TAG" + printf "%-22s %-30s %s\n" "LICENSE_KEY" "" "${LICENSE_KEY:-}" printf "%-22s %-30s %s\n" "TEST_CONN" "127.0.0.1:" "$TEST_CONN" } @@ -378,6 +381,7 @@ Options: --image Image repo (default: anylogco/anylog-network) --node-name Override container name --test-conn REST endpoint for test commands + --license-key License key to inject into container env --manual Use docker run instead of docker compose --no-manual Use docker compose (default) diff --git a/license-generator/license_key.sh b/license-generator/license_key.sh new file mode 100644 index 00000000..9947a0f1 --- /dev/null +++ b/license-generator/license_key.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# license.sh — LICENSE_KEY resolver +# +# Resolves LICENSE_KEY from (in priority order): +# 1. Environment variable (set manually or passed via Makefile) +# 2. Config file (base_configs.env for multi-file layout, +# node_configs.env for single-file layout) +# +# Exports LICENSE_KEY into the environment so both deploy.sh and +# docker compose inherit it. +# +# Usage (always sourced, never executed directly): +# source docker-makefiles/license.sh +# +# Arguments: +# NODE_CONFIGS Canonical node type dir name (e.g. anylog-operator) +# The same value passed as ANYLOG_TYPE elsewhere. + +# ── Guard: must be sourced, not executed ────────────────────────────── +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + echo "ERROR: license.sh must be sourced, not executed directly." >&2 + echo " Use: source docker-makefiles/license.sh " >&2 + exit 1 +fi + +# ── Helpers ─────────────────────────────────────────────────────────── +_license_die() { echo "ERROR: $1" >&2; return 1; } + +# ── Args ────────────────────────────────────────────────────────────── +_license_resolve() { + local node_configs="${1:-}" + local config_file="" + + [[ -n "${node_configs}" ]] \ + || _license_die "license.sh: NODE_CONFIGS argument is required" || return 1 + + # ── 1. Already in environment ──────────────────────────────────────── + if [[ -n "${LICENSE_KEY:-}" ]]; then + echo "LICENSE_KEY: resolved from environment" + export LICENSE_KEY + return 0 + fi + + # ── 2. Locate config file ──────────────────────────────────────────── + # Multi-file layout: LICENSE_KEY lives in base_configs.env + # Single-file layout: LICENSE_KEY lives in node_configs.env + local base_env="docker-makefiles/${node_configs}/base_configs.env" + local single_env="docker-makefiles/${node_configs}/node_configs.env" + + if [[ -f "${base_env}" ]]; then + config_file="${base_env}" + elif [[ -f "${single_env}" ]]; then + config_file="${single_env}" + else + _license_die "license.sh: no config file found for '${node_configs}'" || return 1 + fi + + # ── 3. Read from config file ───────────────────────────────────────── + local key + key=$(grep -m1 '^LICENSE_KEY=' "${config_file}" 2>/dev/null \ + | cut -d= -f2- | tr -d '"'"'"' | tr -d '[:space:]' || true) + + if [[ -n "${key}" ]]; then + echo "LICENSE_KEY: resolved from ${config_file}" + export LICENSE_KEY="${key}" + return 0 + fi + + # ── 4. Not found anywhere ──────────────────────────────────────────── + _license_die "LICENSE_KEY is not set. Provide it via: + - Environment: LICENSE_KEY= make up + - Makefile: LICENSE_KEY := + - Config file: LICENSE_KEY= in ${config_file}" || return 1 +} + +_license_resolve "$@" \ No newline at end of file From 107955becbb3844f889f33267571cd877d6b8a37 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 12 May 2026 21:14:07 +0000 Subject: [PATCH 23/46] chore: update version for os-dev [skip ci] --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 1f23bab5..5fd5d980 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = docker-compose -version = 1.4.2604 (os-dev - 9be036 [2026-05-08 23:48:25]) +version = 1.4.2604 (os-dev - 64ceeb [2026-05-12 21:14:07]) author = AnyLog Co. contact = info@anylog.co description = docker-compose configurations for AnyLog deployment From ed3ea584e624209af05a1dad6d0de4aad1e408a5 Mon Sep 17 00:00:00 2001 From: Ori Shadmon Date: Tue, 12 May 2026 19:53:04 -0700 Subject: [PATCH 24/46] license logic integration - in progress Signed-off-by: Ori Shadmon --- Makefile | 26 +-- deploy.sh | 60 +++--- docker-makefiles/build_docker_compose.sh | 2 +- docker-makefiles/prep_configs.sh | 2 +- license-generator/Dockerfile | 8 +- license-generator/deploy.sh | 6 + license-generator/docker-compose.yml | 5 +- license-generator/license_key.sh | 224 +++++++++++++++++------ 8 files changed, 225 insertions(+), 108 deletions(-) diff --git a/Makefile b/Makefile index 4b7f3990..2b262564 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,6 @@ #!/bin/Makefile $(info LOADING MAKEFILE) -SHELL := /bin/bash - # ────────────────────────────────────────────── # Default values # ────────────────────────────────────────────── @@ -33,8 +31,10 @@ ifneq ($(TEST_CONN),) _FLAGS += --test-conn $(TEST_CONN) endif -ANYLOG_SH := bash -x ./docker-makefiles/deploy.sh +ANYLOG_SH := bash deploy.ori.sh +# ────────────────────────────────────────────── +all: help # ────────────────────────────────────────────── # Docker Hub @@ -51,11 +51,8 @@ pull: ## pull image from Docker Hub dry-run: ## generate docker-compose.yaml (skipped in manual mode) $(ANYLOG_SH) dry-run $(_FLAGS) -# Read LICENSE_KEY from sentinel and export into the environment so deploy.sh -# and docker compose both inherit it — matching the same export pattern used -# by IS_MANUAL, ANYLOG_TYPE, TAG etc. at the top of this file. -up: ## start AnyLog instance (auto-configures rsyslog if SYSLOG_MONITORING=true) - @export LICENSE_KEY=$$(cut -d'|' -f5); $(ANYLOG_SH) up $(_FLAGS) +up: ## start AnyLog instance + $(ANYLOG_SH) up $(_FLAGS) down: ## stop AnyLog instance $(ANYLOG_SH) down $(_FLAGS) @@ -120,22 +117,15 @@ help: ## show this help message @echo " IS_MANUAL Use docker run instead of docker compose (default: false)" @echo " ANYLOG_TYPE Node type: generic master operator query publisher" @echo " standalone-operator standalone-publisher" - @echo " TAG Image tag (default: 1.4.2604)" + @echo " TAG Image tag (default: pre-develop)" @echo " IMAGE Image repository (default: anylogco/anylog-network)" @echo " TEST_CONN ip:port for test commands (default: auto-resolved)" - @echo " LICENSE_KEY License key (prompted on first run, stored in)" - @echo " LICENSE_URL License registration endpoint (default: http://127.0.0.1:8001/api/license-accept)" - @echo " SYSLOG_MONITORING Set to 'true' in node_configs.env to enable syslog → AnyLog forwarding" - @echo " Linux: rsyslog drop-in (TCP) | macOS: syslogd append (UDP)" - @echo " ANYLOG_BROKER_PORT Port syslog forwards to (read from node_configs.env)" @echo "" @echo "Without make:" @echo " bash deploy.sh help" @echo "" -.PHONY: license-check clean-license debug-key debug-post \ - login pull dry-run up down clean clean-all \ +.PHONY: login pull dry-run up down clean clean-all \ logs logs-f attach exec exec-root \ full-test test-status test-node test-network check-processes \ - check-vars help \ - syslog-setup syslog-remove + check-vars help \ No newline at end of file diff --git a/deploy.sh b/deploy.sh index af1d43c3..77121c43 100644 --- a/deploy.sh +++ b/deploy.sh @@ -2,12 +2,18 @@ # deploy.sh — AnyLog node lifecycle manager # Usage: bash deploy.sh [OPTIONS] # Run: bash deploy.sh help -set -euo pipefail +#set -euo pipefail # ────────────────────────────────────────────── # Defaults (override via environment or flags) # ────────────────────────────────────────────── -export LICENSE_KEY="${LICENSE_KEY:-}" +IS_MANUAL="${IS_MANUAL:-false}" +ANYLOG_TYPE="${ANYLOG_TYPE:-anylog-generic}" +TAG="${TAG:-1.4.2604}" +IMAGE="${IMAGE:-anylogco/anylog-network}" +TEST_CONN="${TEST_CONN:-}" +NODE_NAME="${NODE_NAME:-}" +LICENSE_KEY="${LICENSE_KEY:-}" # ────────────────────────────────────────────── # Helpers @@ -87,14 +93,6 @@ _check_configs() { DOCKER_COMPOSE_FILE="docker-makefiles/docker-compose-files/${ANYLOG_TYPE}-docker-compose.yaml" -# remove DOCKER_COMPOSE_FILE if exists -_cmd_clean_compose() { - if [[ "$IS_MANUAL" == "false" ]] && [[ -f ${DOCKER_COMPOSE_FILE} ]]; then - rm -rf "${DOCKER_COMPOSE_FILE}" - fi -} - - # ────────────────────────────────────────────── # Parse flags (--is-manual, --type, --tag, etc.) # ────────────────────────────────────────────── @@ -108,7 +106,6 @@ while [[ $# -gt 0 ]]; do --image) IMAGE="$2"; shift 2 ;; --node-name) NODE_NAME="$2"; shift 2 ;; --test-conn) TEST_CONN="$2"; shift 2 ;; - --license-key) export LICENSE_KEY="$2"; shift 2 ;; --manual) IS_MANUAL="true"; shift ;; --no-manual) IS_MANUAL="false"; shift ;; *) die "Unknown option: $1" ;; @@ -133,7 +130,6 @@ cmd_login() { cmd_pull() { _check_configs _load_configs - bash license-generator/license_key.sh "${ANYLOG_TYPE}" echo "Pulling ${IMAGE}:${TAG}..." ${CONTAINER_CMD} pull "docker.io/${IMAGE}:${TAG}" } @@ -178,8 +174,11 @@ cmd_dry_run() { cmd_up() { cmd_dry_run - echo "LICENSE_KEY: ${ANYLOG_TYPE}" - bash docker-makefiles/license.sh "${ANYLOG_TYPE}" + cmd_license_check + + local license_flag="" + [[ -n "${LICENSE_KEY}" ]] && license_flag="-e LICENSE_KEY=${LICENSE_KEY}" + if [[ "$IS_MANUAL" == "true" ]]; then echo "Deploying ${ANYLOG_TYPE} [manual / docker run]" local single_file="docker-makefiles/${ANYLOG_TYPE}/node_configs.env" @@ -193,8 +192,6 @@ cmd_up() { vol_scripts="-v ${NODE_NAME}-local-scripts:/app/deployment-scripts" fi - local license_flag="" - [[ -n "$LICENSE_KEY" ]] && license_flag="-e LICENSE_KEY=${LICENSE_KEY}" ${CONTAINER_CMD} run -it -d --detach-keys=ctrl-d \ --name "${NODE_NAME}" \ --network host \ @@ -208,10 +205,11 @@ cmd_up() { "${IMAGE}:${TAG}" else echo "Deploying ${ANYLOG_TYPE}" - LICENSE_KEY="${LICENSE_KEY}" ${DOCKER_COMPOSE_CMD} -f "${DOCKER_COMPOSE_FILE}" up -d + ${DOCKER_COMPOSE_CMD} -f "${DOCKER_COMPOSE_FILE}" up -d ${license_flag} fi } + cmd_down() { _check_configs _load_configs @@ -225,7 +223,6 @@ cmd_down() { fi } - cmd_clean() { _check_configs _load_configs @@ -242,7 +239,6 @@ cmd_clean() { echo "Stopping + removing volumes: ${ANYLOG_TYPE}" ${DOCKER_COMPOSE_CMD} -f "${DOCKER_COMPOSE_FILE}" down -v fi - _cmd_clean_compose } cmd_clean_all() { @@ -262,7 +258,6 @@ cmd_clean_all() { echo "Stopping + removing volumes + image: ${ANYLOG_TYPE}" ${DOCKER_COMPOSE_CMD} -f "${DOCKER_COMPOSE_FILE}" down -v --rmi all fi - _cmd_clean_compose } cmd_logs() { @@ -290,6 +285,29 @@ cmd_exec_root() { ${CONTAINER_CMD} exec -u root -it "${NODE_NAME}" /bin/bash } +# ────────────────────────────────────────────── +# License Key logic +# ────────────────────────────────────────────── +cmd_license_check() { + local env_file="docker-makefiles/${ANYLOG_TYPE}/.env" + local single_file="docker-makefiles/${ANYLOG_TYPE}/node_configs.env" + + if [[ -z "${LICENSE_KEY}" ]] && [[ -f "${env_file}" ]]; then + LICENSE_KEY=$(grep -m1 '^LICENSE_KEY=' "$env_file" | cut -d= -f2- | tr -d '"\r') + elif [[ -z "${LICENSE_KEY}" ]] && [[ -f "${single_file}" ]]; then + LICENSE_KEY=$(grep -m1 '^LICENSE_KEY=' "$single_file" | cut -d= -f2- | tr -d '"\r') + fi + + if [[ -z "${LICENSE_KEY}" ]] ; then + die "Missing license key, cannot deploy AnyLog." + fi + + export LICENSE_KEY + echo ${LICENSE_KEY} + bash ./license-generator/license_key.sh "${LICENSE_KEY}" +} + + # ────────────────────────────────────────────── # Testing # ────────────────────────────────────────────── @@ -344,7 +362,6 @@ cmd_check_vars() { printf "%-22s %-30s %s\n" "IMAGE" "anylogco/anylog-network" "$IMAGE" printf "%-22s %-30s %s\n" "NODE_NAME" "" "${NODE_NAME:-}" printf "%-22s %-30s %s\n" "TAG" "pre-develop" "$TAG" - printf "%-22s %-30s %s\n" "LICENSE_KEY" "" "${LICENSE_KEY:-}" printf "%-22s %-30s %s\n" "TEST_CONN" "127.0.0.1:" "$TEST_CONN" } @@ -381,7 +398,6 @@ Options: --image Image repo (default: anylogco/anylog-network) --node-name Override container name --test-conn REST endpoint for test commands - --license-key License key to inject into container env --manual Use docker run instead of docker compose --no-manual Use docker compose (default) diff --git a/docker-makefiles/build_docker_compose.sh b/docker-makefiles/build_docker_compose.sh index 34137453..b83be99c 100644 --- a/docker-makefiles/build_docker_compose.sh +++ b/docker-makefiles/build_docker_compose.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -euo pipefail +#set -euo pipefail # -------- Helpers -------- die() { diff --git a/docker-makefiles/prep_configs.sh b/docker-makefiles/prep_configs.sh index 5d9a0d2c..c3c4f30a 100644 --- a/docker-makefiles/prep_configs.sh +++ b/docker-makefiles/prep_configs.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -euo pipefail +#set -euo pipefail # -------- Helpers -------- die() { diff --git a/license-generator/Dockerfile b/license-generator/Dockerfile index 5a33e20d..149df1fb 100644 --- a/license-generator/Dockerfile +++ b/license-generator/Dockerfile @@ -1,13 +1,13 @@ -FROM python:3.13-slim +FROM python:3.12-slim WORKDIR /app COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt - COPY src/license-server.py app.py -RUN mkdir -p /data +RUN apt-get -y update && \ + pip install --no-cache-dir -r requirements.txt && \ + mkdir -p /data EXPOSE 8001 diff --git a/license-generator/deploy.sh b/license-generator/deploy.sh index 8efb2985..55cf9c15 100644 --- a/license-generator/deploy.sh +++ b/license-generator/deploy.sh @@ -7,6 +7,12 @@ set -euo pipefail # ────────────────────────────────────────────── # Defaults (override via environment or flags) # ────────────────────────────────────────────── +IS_MANUAL="${IS_MANUAL:-false}" +ANYLOG_TYPE="${ANYLOG_TYPE:-anylog-generic}" +TAG="${TAG:-pre-develop}" +IMAGE="${IMAGE:-anylogco/anylog-network}" +TEST_CONN="${TEST_CONN:-}" +NODE_NAME="${NODE_NAME:-}" export LICENSE_KEY="${LICENSE_KEY:-}" # ────────────────────────────────────────────── diff --git a/license-generator/docker-compose.yml b/license-generator/docker-compose.yml index aa1390a6..7178af4c 100644 --- a/license-generator/docker-compose.yml +++ b/license-generator/docker-compose.yml @@ -10,4 +10,7 @@ services: environment: DATA_DIR: /data volumes: - - ./shared-license-data:/data \ No newline at end of file + - license-data:/data + +volumes: + license-data: \ No newline at end of file diff --git a/license-generator/license_key.sh b/license-generator/license_key.sh index 9947a0f1..e1368614 100644 --- a/license-generator/license_key.sh +++ b/license-generator/license_key.sh @@ -1,76 +1,178 @@ #!/usr/bin/env bash -# license.sh — LICENSE_KEY resolver +# license_key.sh — license acceptance form and registration # -# Resolves LICENSE_KEY from (in priority order): -# 1. Environment variable (set manually or passed via Makefile) -# 2. Config file (base_configs.env for multi-file layout, -# node_configs.env for single-file layout) +# Usage: +# bash license-generator/license_key.sh [LICENSE_KEY] # -# Exports LICENSE_KEY into the environment so both deploy.sh and -# docker compose inherit it. +# If LICENSE_KEY is provided as $1, skips the key prompt. +# If not provided, prompts the user to enter it. # -# Usage (always sourced, never executed directly): -# source docker-makefiles/license.sh -# -# Arguments: -# NODE_CONFIGS Canonical node type dir name (e.g. anylog-operator) -# The same value passed as ANYLOG_TYPE elsewhere. - -# ── Guard: must be sourced, not executed ────────────────────────────── -if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then - echo "ERROR: license.sh must be sourced, not executed directly." >&2 - echo " Use: source docker-makefiles/license.sh " >&2 - exit 1 +# Environment: +# LICENSE_URL License server endpoint (default: http://23.239.12.151:8001/api/license-accept) + +LICENSE_KEY="${1:-}" +LICENSE_URL="${LICENSE_URL:-http://23.239.12.151:8001/api/license-accept}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# ── Show license agreement ──────────────────────────────────────────── +if [[ -f "${SCRIPT_DIR}/../LICENSE.txt" ]]; then + more "${SCRIPT_DIR}/../LICENSE.txt" fi -# ── Helpers ─────────────────────────────────────────────────────────── -_license_die() { echo "ERROR: $1" >&2; return 1; } +echo "" +echo "========================================" +echo " License Agreement Acceptance Required" +echo "========================================" +echo "" -# ── Args ────────────────────────────────────────────────────────────── -_license_resolve() { - local node_configs="${1:-}" - local config_file="" +# ── Smart quote helpers ─────────────────────────────────────────────── +_LQ=$(printf '\342\200\234') +_RQ=$(printf '\342\200\235') - [[ -n "${node_configs}" ]] \ - || _license_die "license.sh: NODE_CONFIGS argument is required" || return 1 +# ── Validate license key format ─────────────────────────────────────── +_validate_key() { + local key="$1" + echo "${key}" | awk -v lq="${_LQ}" -v rq="${_RQ}" '{ + brace = index($0, "{") + if (brace == 0) { print "INVALID|no brace in key"; exit } + hex = substr($0, 1, brace - 1) + json = substr($0, brace) + gsub(/\\"/, "\"", json); gsub(lq, "\"", json); gsub(rq, "\"", json) + hex_ok = (length(hex) == 256 && hex ~ /^[0-9a-f]/) + co_ok = (json ~ /"company":"[^"]+"/ ) + ex_ok = (json ~ /"expiration":"[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]"/) + ty_ok = (json ~ /"type":"[^"]+"/ ) + if (hex_ok && co_ok && ex_ok && ty_ok) { print "VALID"; exit } + msg = "INVALID|" + if (!hex_ok) msg = msg "hex must be 256 lowercase hex chars (got " length(hex) "); " + if (!co_ok) msg = msg "missing/invalid company; " + if (!ex_ok) msg = msg "missing/invalid expiration YYYY-MM-DD; " + if (!ty_ok) msg = msg "missing/invalid type; " + print msg; exit + }' +} - # ── 1. Already in environment ──────────────────────────────────────── - if [[ -n "${LICENSE_KEY:-}" ]]; then - echo "LICENSE_KEY: resolved from environment" - export LICENSE_KEY - return 0 - fi +# ── Parse field from license key JSON ──────────────────────────────── +_parse_key() { + local field="$1" key="$2" + echo "${key}" | awk -v lq="${_LQ}" -v rq="${_RQ}" -v field="${field}" '{ + brace = index($0, "{") + if (brace == 0) { print ""; exit } + json = substr($0, brace) + gsub(/\\"/, "\"", json); gsub(lq, "\"", json); gsub(rq, "\"", json) + pattern = "\"" field "\":\"[^\"]*\"" + match(json, pattern) + if (RSTART) { + val = substr(json, RSTART, RLENGTH) + gsub(/.*":"/, "", val); gsub(/".*/, "", val) + print val + } + }' +} - # ── 2. Locate config file ──────────────────────────────────────────── - # Multi-file layout: LICENSE_KEY lives in base_configs.env - # Single-file layout: LICENSE_KEY lives in node_configs.env - local base_env="docker-makefiles/${node_configs}/base_configs.env" - local single_env="docker-makefiles/${node_configs}/node_configs.env" - - if [[ -f "${base_env}" ]]; then - config_file="${base_env}" - elif [[ -f "${single_env}" ]]; then - config_file="${single_env}" - else - _license_die "license.sh: no config file found for '${node_configs}'" || return 1 +# ── Prompt for license key if not provided ──────────────────────────── +if [[ -z "${LICENSE_KEY}" ]]; then + while true; do + printf "License Key: "; read -r LICENSE_KEY &2 + exit 1 fi +fi - # ── 3. Read from config file ───────────────────────────────────────── - local key - key=$(grep -m1 '^LICENSE_KEY=' "${config_file}" 2>/dev/null \ - | cut -d= -f2- | tr -d '"'"'"' | tr -d '[:space:]' || true) +# ── Extract info from key ───────────────────────────────────────────── +COMPANY=$(_parse_key "company" "${LICENSE_KEY}") +TYPE=$(_parse_key "type" "${LICENSE_KEY}") +EXPIRATION=$(_parse_key "expiration" "${LICENSE_KEY}") - if [[ -n "${key}" ]]; then - echo "LICENSE_KEY: resolved from ${config_file}" - export LICENSE_KEY="${key}" - return 0 - fi +echo "" +echo "License key recognized:" +echo " Company: ${COMPANY}" +echo " Type: ${TYPE}" +echo " Expires: ${EXPIRATION}" +echo "" - # ── 4. Not found anywhere ──────────────────────────────────────────── - _license_die "LICENSE_KEY is not set. Provide it via: - - Environment: LICENSE_KEY= make up - - Makefile: LICENSE_KEY := - - Config file: LICENSE_KEY= in ${config_file}" || return 1 -} +# ── Collect user info ───────────────────────────────────────────────── +while true; do + printf "Full Name: "; read -r NAME /tmp/license_payload.json + +HTTP_CODE=$(curl -sk -o /tmp/license_response.txt -w "%{http_code}" \ + -X POST "${LICENSE_URL}" \ + -H "Content-Type: application/json" \ + -d @/tmp/license_payload.json) || true + +if [[ "${HTTP_CODE}" == "200" ]] || [[ "${HTTP_CODE}" == "201" ]]; then + echo "Registration successful." + echo "${NAME}|${COMPANY}|${EMAIL}|${PROJECT}|${LICENSE_KEY}|${TS}" > .license_accepted +else + echo "We could not complete license registration right now." + echo "Please contact support@anylog.co with your license information." + printf "Continue anyway? [yes/no]: "; read -r CONT .license_accepted +fi \ No newline at end of file From 4392b97a2cf6cbc875cd8d12c8615a66b2b9399a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 13 May 2026 02:53:45 +0000 Subject: [PATCH 25/46] chore: update version for os-dev [skip ci] --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 5fd5d980..1232ecea 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = docker-compose -version = 1.4.2604 (os-dev - 64ceeb [2026-05-12 21:14:07]) +version = 1.4.2604 (os-dev - 1a804f [2026-05-13 02:53:45]) author = AnyLog Co. contact = info@anylog.co description = docker-compose configurations for AnyLog deployment From a0614f3046ac9cd32d7332217084d9ab414862c9 Mon Sep 17 00:00:00 2001 From: Ori Shadmon Date: Tue, 12 May 2026 20:05:34 -0700 Subject: [PATCH 26/46] license key logic interation Signed-off-by: Ori Shadmon --- Makefile | 2 +- deploy.sh | 4 +- license-generator/license_key.sh | 89 +++++++++++++++++++------------- 3 files changed, 57 insertions(+), 38 deletions(-) diff --git a/Makefile b/Makefile index 2b262564..24100e59 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ ifneq ($(TEST_CONN),) _FLAGS += --test-conn $(TEST_CONN) endif -ANYLOG_SH := bash deploy.ori.sh +ANYLOG_SH := bash deploy.sh # ────────────────────────────────────────────── all: help diff --git a/deploy.sh b/deploy.sh index 77121c43..60a8b2d4 100644 --- a/deploy.sh +++ b/deploy.sh @@ -176,6 +176,7 @@ cmd_up() { cmd_dry_run cmd_license_check + # -e flag is only valid for `docker run`; docker compose reads LICENSE_KEY from the env file local license_flag="" [[ -n "${LICENSE_KEY}" ]] && license_flag="-e LICENSE_KEY=${LICENSE_KEY}" @@ -205,7 +206,7 @@ cmd_up() { "${IMAGE}:${TAG}" else echo "Deploying ${ANYLOG_TYPE}" - ${DOCKER_COMPOSE_CMD} -f "${DOCKER_COMPOSE_FILE}" up -d ${license_flag} + ${DOCKER_COMPOSE_CMD} -f "${DOCKER_COMPOSE_FILE}" up -d fi } @@ -303,7 +304,6 @@ cmd_license_check() { fi export LICENSE_KEY - echo ${LICENSE_KEY} bash ./license-generator/license_key.sh "${LICENSE_KEY}" } diff --git a/license-generator/license_key.sh b/license-generator/license_key.sh index e1368614..444005ae 100644 --- a/license-generator/license_key.sh +++ b/license-generator/license_key.sh @@ -10,21 +10,15 @@ # Environment: # LICENSE_URL License server endpoint (default: http://23.239.12.151:8001/api/license-accept) +# ── Skip if already accepted ────────────────────────────────────────── +if [[ -f ".license_accepted" ]]; then + exit 0 +fi + LICENSE_KEY="${1:-}" LICENSE_URL="${LICENSE_URL:-http://23.239.12.151:8001/api/license-accept}" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# ── Show license agreement ──────────────────────────────────────────── -if [[ -f "${SCRIPT_DIR}/../LICENSE.txt" ]]; then - more "${SCRIPT_DIR}/../LICENSE.txt" -fi - -echo "" -echo "========================================" -echo " License Agreement Acceptance Required" -echo "========================================" -echo "" - # ── Smart quote helpers ─────────────────────────────────────────────── _LQ=$(printf '\342\200\234') _RQ=$(printf '\342\200\235') @@ -38,7 +32,7 @@ _validate_key() { hex = substr($0, 1, brace - 1) json = substr($0, brace) gsub(/\\"/, "\"", json); gsub(lq, "\"", json); gsub(rq, "\"", json) - hex_ok = (length(hex) == 256 && hex ~ /^[0-9a-f]/) + hex_ok = (length(hex) == 256 && hex ~ /^[0-9a-f]+$/) co_ok = (json ~ /"company":"[^"]+"/ ) ex_ok = (json ~ /"expiration":"[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]"/) ty_ok = (json ~ /"type":"[^"]+"/ ) @@ -53,6 +47,8 @@ _validate_key() { } # ── Parse field from license key JSON ──────────────────────────────── +# Extracts the value of a named field from the JSON suffix of the key. +# Uses index-based substr to avoid greedy-match issues with gsub. _parse_key() { local field="$1" key="$2" echo "${key}" | awk -v lq="${_LQ}" -v rq="${_RQ}" -v field="${field}" '{ @@ -60,16 +56,31 @@ _parse_key() { if (brace == 0) { print ""; exit } json = substr($0, brace) gsub(/\\"/, "\"", json); gsub(lq, "\"", json); gsub(rq, "\"", json) - pattern = "\"" field "\":\"[^\"]*\"" - match(json, pattern) - if (RSTART) { - val = substr(json, RSTART, RLENGTH) - gsub(/.*":"/, "", val); gsub(/".*/, "", val) - print val - } + # Build search needle: "field":" + needle = "\"" field "\":\"" + pos = index(json, needle) + if (pos == 0) { print ""; exit } + # Advance past the needle to the start of the value + val_start = pos + length(needle) + rest = substr(json, val_start) + # Value ends at the first unescaped double-quote + end_pos = index(rest, "\"") + if (end_pos == 0) { print ""; exit } + print substr(rest, 1, end_pos - 1) }' } +# ── Show license agreement ──────────────────────────────────────────── +if [[ -f "${SCRIPT_DIR}/../LICENSE.txt" ]]; then + more "${SCRIPT_DIR}/../LICENSE.txt" +fi + +echo "" +echo "========================================" +echo " License Agreement Acceptance Required" +echo "========================================" +echo "" + # ── Prompt for license key if not provided ──────────────────────────── if [[ -z "${LICENSE_KEY}" ]]; then while true; do @@ -103,24 +114,32 @@ echo " Type: ${TYPE}" echo " Expires: ${EXPIRATION}" echo "" -# ── Collect user info ───────────────────────────────────────────────── -while true; do - printf "Full Name: "; read -r NAME /dev/null; } +if ! _tty_available 2>/dev/null; then + NAME="Unknown" + EMAIL="Unknown" + PROJECT="Unknown" +else + # ── Collect user info ─────────────────────────────────────────────── + while true; do + printf "Full Name: "; read -r NAME Date: Wed, 13 May 2026 03:05:52 +0000 Subject: [PATCH 27/46] chore: update version for os-dev [skip ci] --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 1232ecea..4f692a41 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = docker-compose -version = 1.4.2604 (os-dev - 1a804f [2026-05-13 02:53:45]) +version = 1.4.2604 (os-dev - d6daac [2026-05-13 03:05:52]) author = AnyLog Co. contact = info@anylog.co description = docker-compose configurations for AnyLog deployment From a55183085444840bdff908d543e808b7c0f8b510 Mon Sep 17 00:00:00 2001 From: Ori Shadmon Date: Tue, 12 May 2026 20:17:00 -0700 Subject: [PATCH 28/46] cleanup to match mark's config files Signed-off-by: Ori Shadmon --- customers/dados.proto | 19 -- customers/dados_pb2.py | 42 --- customers/dados_pb2_grpc.py | 97 ------- customers/grpc_client.al | 14 - .../anylog-generic/node_configs.env | 2 +- .../anylog-master/node_configs.env | 2 +- .../anylog-operator/node_configs.env | 2 +- .../anylog-publisher/node_configs.env | 2 +- .../anylog-query/node_configs.env | 2 +- .../node_configs.env | 47 +-- .../node_configs.env | 2 +- openbao/docker-compose.yml | 24 -- openbao/get4bao.sh | 6 - openbao/openbao.env | 9 - openbao/openbao_cmds.py | 270 ------------------ openbao/run_openbao.sh | 7 - row_count.sh | 33 --- 17 files changed, 16 insertions(+), 564 deletions(-) delete mode 100644 customers/dados.proto delete mode 100644 customers/dados_pb2.py delete mode 100644 customers/dados_pb2_grpc.py delete mode 100644 customers/grpc_client.al delete mode 100644 openbao/docker-compose.yml delete mode 100644 openbao/get4bao.sh delete mode 100644 openbao/openbao.env delete mode 100644 openbao/openbao_cmds.py delete mode 100644 openbao/run_openbao.sh delete mode 100644 row_count.sh diff --git a/customers/dados.proto b/customers/dados.proto deleted file mode 100644 index d05dbaf5..00000000 --- a/customers/dados.proto +++ /dev/null @@ -1,19 +0,0 @@ -syntax = "proto3"; -package persistence; - -service PersistenceService { - rpc WatchTable (WatchRequest) returns (stream DataRow); -} - -message WatchRequest { - string table_id = 1; -} - -message DataRow { - string data = 1; -} - -// missing reply function -message ReplyMessage { - int32 Retval = 1; -} diff --git a/customers/dados_pb2.py b/customers/dados_pb2.py deleted file mode 100644 index 1f41d821..00000000 --- a/customers/dados_pb2.py +++ /dev/null @@ -1,42 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# NO CHECKED-IN PROTOBUF GENCODE -# source: dados.proto -# Protobuf Python Version: 6.31.1 -"""Generated protocol buffer code.""" -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import runtime_version as _runtime_version -from google.protobuf import symbol_database as _symbol_database -from google.protobuf.internal import builder as _builder -_runtime_version.ValidateProtobufRuntimeVersion( - _runtime_version.Domain.PUBLIC, - 6, - 31, - 1, - '', - 'dados.proto' -) -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0b\x64\x61\x64os.proto\x12\x0bpersistence\" \n\x0cWatchRequest\x12\x10\n\x08table_id\x18\x01 \x01(\t\"\x17\n\x07\x44\x61taRow\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\t\"\x1e\n\x0cReplyMessage\x12\x0e\n\x06Retval\x18\x01 \x01(\x05\x32U\n\x12PersistenceService\x12?\n\nWatchTable\x12\x19.persistence.WatchRequest\x1a\x14.persistence.DataRow0\x01\x62\x06proto3') - -_globals = globals() -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'dados_pb2', _globals) -if not _descriptor._USE_C_DESCRIPTORS: - DESCRIPTOR._loaded_options = None - _globals['_WATCHREQUEST']._serialized_start=28 - _globals['_WATCHREQUEST']._serialized_end=60 - _globals['_DATAROW']._serialized_start=62 - _globals['_DATAROW']._serialized_end=85 - _globals['_REPLYMESSAGE']._serialized_start=87 - _globals['_REPLYMESSAGE']._serialized_end=117 - _globals['_PERSISTENCESERVICE']._serialized_start=119 - _globals['_PERSISTENCESERVICE']._serialized_end=204 -# @@protoc_insertion_point(module_scope) diff --git a/customers/dados_pb2_grpc.py b/customers/dados_pb2_grpc.py deleted file mode 100644 index d93c3bf0..00000000 --- a/customers/dados_pb2_grpc.py +++ /dev/null @@ -1,97 +0,0 @@ -# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! -"""Client and server classes corresponding to protobuf-defined services.""" -import grpc -import warnings - -import dados_pb2 as dados__pb2 - -GRPC_GENERATED_VERSION = '1.76.0' -GRPC_VERSION = grpc.__version__ -_version_not_supported = False - -try: - from grpc._utilities import first_version_is_lower - _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) -except ImportError: - _version_not_supported = True - -if _version_not_supported: - raise RuntimeError( - f'The grpc package installed is at version {GRPC_VERSION},' - + ' but the generated code in dados_pb2_grpc.py depends on' - + f' grpcio>={GRPC_GENERATED_VERSION}.' - + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' - + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' - ) - - -class PersistenceServiceStub(object): - """Missing associated documentation comment in .proto file.""" - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.WatchTable = channel.unary_stream( - '/persistence.PersistenceService/WatchTable', - request_serializer=dados__pb2.WatchRequest.SerializeToString, - response_deserializer=dados__pb2.DataRow.FromString, - _registered_method=True) - - -class PersistenceServiceServicer(object): - """Missing associated documentation comment in .proto file.""" - - def WatchTable(self, request, context): - """Missing associated documentation comment in .proto file.""" - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - -def add_PersistenceServiceServicer_to_server(servicer, server): - rpc_method_handlers = { - 'WatchTable': grpc.unary_stream_rpc_method_handler( - servicer.WatchTable, - request_deserializer=dados__pb2.WatchRequest.FromString, - response_serializer=dados__pb2.DataRow.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - 'persistence.PersistenceService', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler,)) - server.add_registered_method_handlers('persistence.PersistenceService', rpc_method_handlers) - - - # This class is part of an EXPERIMENTAL API. -class PersistenceService(object): - """Missing associated documentation comment in .proto file.""" - - @staticmethod - def WatchTable(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_stream( - request, - target, - '/persistence.PersistenceService/WatchTable', - dados__pb2.WatchRequest.SerializeToString, - dados__pb2.DataRow.FromString, - options, - channel_credentials, - insecure, - call_credentials, - compression, - wait_for_ready, - timeout, - metadata, - _registered_method=True) diff --git a/customers/grpc_client.al b/customers/grpc_client.al deleted file mode 100644 index 0b26a44b..00000000 --- a/customers/grpc_client.al +++ /dev/null @@ -1,14 +0,0 @@ - \ No newline at end of file diff --git a/docker-makefiles/anylog-generic/node_configs.env b/docker-makefiles/anylog-generic/node_configs.env index 2af268ca..87b68ed1 100644 --- a/docker-makefiles/anylog-generic/node_configs.env +++ b/docker-makefiles/anylog-generic/node_configs.env @@ -282,7 +282,7 @@ BUCKET_SECRET_KEY="" # Whether to enable Remote-GUI in deployment ENABLE_REMOTE_GUI=false # Remote-GUI version Tag -REMOTE_GUI_TAG=beta2 +REMOTE_GUI_TAG=1.1.0 # NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 REMOTE_GUI_NIC="" # Remote GUI Frontend Port diff --git a/docker-makefiles/anylog-master/node_configs.env b/docker-makefiles/anylog-master/node_configs.env index f5dd3bcb..86c7f1e3 100644 --- a/docker-makefiles/anylog-master/node_configs.env +++ b/docker-makefiles/anylog-master/node_configs.env @@ -281,7 +281,7 @@ BUCKET_SECRET_KEY="" # Whether to enable Remote-GUI in deployment ENABLE_REMOTE_GUI=false # Remote-GUI version Tag -REMOTE_GUI_TAG=beta2 +REMOTE_GUI_TAG=1.1.0 # NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 REMOTE_GUI_NIC="" # Remote GUI Frontend Port diff --git a/docker-makefiles/anylog-operator/node_configs.env b/docker-makefiles/anylog-operator/node_configs.env index 1edc8566..28fa423e 100644 --- a/docker-makefiles/anylog-operator/node_configs.env +++ b/docker-makefiles/anylog-operator/node_configs.env @@ -282,7 +282,7 @@ BUCKET_SECRET_KEY="" # Whether to enable Remote-GUI in deployment ENABLE_REMOTE_GUI=false # Remote-GUI version Tag -REMOTE_GUI_TAG=beta2 +REMOTE_GUI_TAG=1.1.0 # NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 REMOTE_GUI_NIC="" # Remote GUI Frontend Port diff --git a/docker-makefiles/anylog-publisher/node_configs.env b/docker-makefiles/anylog-publisher/node_configs.env index e49a862b..69971810 100644 --- a/docker-makefiles/anylog-publisher/node_configs.env +++ b/docker-makefiles/anylog-publisher/node_configs.env @@ -282,7 +282,7 @@ BUCKET_SECRET_KEY="" # Whether to enable Remote-GUI in deployment ENABLE_REMOTE_GUI=false # Remote-GUI version Tag -REMOTE_GUI_TAG=beta2 +REMOTE_GUI_TAG=1.1.0 # NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 REMOTE_GUI_NIC="" # Remote GUI Frontend Port diff --git a/docker-makefiles/anylog-query/node_configs.env b/docker-makefiles/anylog-query/node_configs.env index fc7bc875..305e80c1 100644 --- a/docker-makefiles/anylog-query/node_configs.env +++ b/docker-makefiles/anylog-query/node_configs.env @@ -282,7 +282,7 @@ BUCKET_SECRET_KEY="" # Whether to enable Remote-GUI in deployment ENABLE_REMOTE_GUI=true # Remote-GUI version Tag -REMOTE_GUI_TAG=beta2 +REMOTE_GUI_TAG=1.1.0 # NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 REMOTE_GUI_NIC="" # Remote GUI Frontend Port diff --git a/docker-makefiles/anylog-standalone-operator/node_configs.env b/docker-makefiles/anylog-standalone-operator/node_configs.env index ebb760b7..0900f1ee 100644 --- a/docker-makefiles/anylog-standalone-operator/node_configs.env +++ b/docker-makefiles/anylog-standalone-operator/node_configs.env @@ -12,38 +12,17 @@ IMAGE=anylogco/anylog-network # Github repo to use for configuring / deploying node # The path can either be a Github URL and then the code will automatically clone # the repo if needed OR a local path for deployment-scripts and then use that instead of cloning into the repositroy +DEPLOYMENTS_REPO=https://github.com/AnyLog-co/deployment-scripts # Branch associated with git repo -DEPLOYMENTS_REPO="/mnt/c/Users/oshad/AnyLog-code/deployment-scripts" DEPLOYMENTS_BRANCH="os-dev" # specify the docker socket - to be used when deploying docker monitoring DOCKER_SOCKET="/var/run/docker.sock" -# Local folders to mount inside the AnyLog container. -# -# Examples: -# No volumes: -# USER_VOLUMES="" -# -# Single volume: -# USER_VOLUMES="/home/user/grpc-scripts" -# -# Multiple volumes (space-separated): -# USER_VOLUMES="/home/user/grpc-scripts /home/user/local-deployments" -# -# Notes: -# - Use absolute paths only -# - Paths are mounted automatically under /app/ -# -# Example: -# /home/user/grpc-scripts -> /app/grpc-scripts -# -USER_VOLUMES="/mnt/c/Users/oshad/AnyLog-code/docker-compose/customers" - #==== basic ==== # Core node configuration - required for all deployments #--- General --- -# Configuration based deployment node deployment type +# Configuration based deploymnet node deployment type # Node Types: # generic: sandbox with only network configured # master: "blockchain" emulator @@ -109,11 +88,10 @@ BLOCKCHAIN_DESTINATION=file # Owner of the cluster CLUSTER_NAME=anylog-cluster1 # Logical database name -DEFAULT_DBMS=persistence +DEFAULT_DBMS=new_company #==== southbound ==== # Data ingestion protocols (MQTT, OPC-UA, video streaming, monitoring) - #--- Monitoring --- # Whether to monitor the node or not NODE_MONITORING=true @@ -124,9 +102,6 @@ SYSLOG_MONITORING=true # Docker Monitoring DOCKER_MONITORING=true -# how often data is being pulled from docker -DOCKER_FREQUENCY = 30 - #--- Video Streaming --- # Enable or disable video streaming. # true = stream video, false = do not stream @@ -145,7 +120,7 @@ YOLO_MODEL_PORT=50051 #--- MQTT --- # Whether to enable the default MQTT process -ENABLE_MQTT=false +ENABLE_MQTT=true # IP address of MQTT broker MQTT_BROKER=172.104.228.251 @@ -218,7 +193,7 @@ DNS_DOMAIN="" DNS="" # Bind settings - a true value binds to a specific IP and Port (a false value binds to all IPs) -TCP_BIND=false +TCP_BIND=true REST_BIND=false BROKER_BIND=false @@ -258,20 +233,18 @@ ENABLE_HA=false START_DATE=30 # How many threads to use in the operator process OPERATOR_THREADS=3 -# Parallel threads to be used in order digest data into AnyLog -OPERATOR_HELPERS=5 + #--- Sub processes --- # Compress JSON and SQL file(s) backup COMPRESS_FILE=true # When data comes in write to database immediately, as opposed to waiting for a full buffer WRITE_IMMEDIATE=true # If buffer is not full, how long to wait until pushing data through -THRESHOLD_TIME=10 seconds +THRESHOLD_TIME=60 seconds # Buffer size to reach, at which point data is pushed through -THRESHOLD_VOLUME=1KB +THRESHOLD_VOLUME=10KB # Number of parallel queries QUERY_POOL=6 -# for `run operator` process, specify the number of parallel threads to use #==== secrets ==== # Sensitive credentials - do not commit to version control @@ -306,9 +279,9 @@ BUCKET_SECRET_KEY="" #==== remote-gui ==== # Companion UI application - deployed alongside the AnyLog node # Whether to enable Remote-GUI in deployment -ENABLE_REMOTE_GUI=false +ENABLE_REMOTE_GUI=true # Remote-GUI version Tag -REMOTE_GUI_TAG=beta2 +REMOTE_GUI_TAG=1.1.0 # NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 REMOTE_GUI_NIC="" # Remote GUI Frontend Port diff --git a/docker-makefiles/anylog-standalone-publisher/node_configs.env b/docker-makefiles/anylog-standalone-publisher/node_configs.env index 78003efb..20c673e8 100644 --- a/docker-makefiles/anylog-standalone-publisher/node_configs.env +++ b/docker-makefiles/anylog-standalone-publisher/node_configs.env @@ -281,7 +281,7 @@ BUCKET_SECRET_KEY="" # Whether to enable Remote-GUI in deployment ENABLE_REMOTE_GUI=true # Remote-GUI version Tag -REMOTE_GUI_TAG=beta2 +REMOTE_GUI_TAG=1.1.0 # NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 REMOTE_GUI_NIC="" # Remote GUI Frontend Port diff --git a/openbao/docker-compose.yml b/openbao/docker-compose.yml deleted file mode 100644 index 5b195380..00000000 --- a/openbao/docker-compose.yml +++ /dev/null @@ -1,24 +0,0 @@ -services: - anylog-node: - image: anylogco/anylog-network:1.3.2412-beta8 - restart: always - container_name: anylog-node - stdin_open: true - tty: true - network_mode: host - environment: - - INIT_TYPE=prod - - OPENBAO_VAULT_IP=http://0.0.0.0:8200 - - OPENBAO_VAULT_TOKEN=s.57hG4fLMZzRbbPF7mF5fA2Md - - SECTION_NAME=mybao - volumes: - - anylog-node-anylog:/app/AnyLog-Network/anylog - - anylog-node-blockchain:/app/AnyLog-Network/blockchain - - anylog-node-data:/app/AnyLog-Network/data - - anylog-node-local-scripts:/app/deployment-scripts - -volumes: - anylog-node-anylog: - anylog-node-blockchain: - anylog-node-data: - anylog-node-local-scripts: \ No newline at end of file diff --git a/openbao/get4bao.sh b/openbao/get4bao.sh deleted file mode 100644 index 9a2629b8..00000000 --- a/openbao/get4bao.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -# 0.0.0.0:8200 s.Pb8yAsBHAKyva0DHmQA4Ib12 -python3 get4bao.py ${VAULT_URL} ${VAULT_TOKEN} - -source $ANYLOG_PATH/openbao/configs.env \ No newline at end of file diff --git a/openbao/openbao.env b/openbao/openbao.env deleted file mode 100644 index 9bd4ed5f..00000000 --- a/openbao/openbao.env +++ /dev/null @@ -1,9 +0,0 @@ -# test configs -SERVICE_PORT=32548 -TCP_BIND=true -TCP_THREADS=3 - -REST_PORT=32549 -REST_TIMEOUT=30 -REST_BIND=false -REST_THREADS=3 \ No newline at end of file diff --git a/openbao/openbao_cmds.py b/openbao/openbao_cmds.py deleted file mode 100644 index 3b246854..00000000 --- a/openbao/openbao_cmds.py +++ /dev/null @@ -1,270 +0,0 @@ -# URL: https://openbao.org/api-docs/ -import argparse -import ast -import json -import os -import requests - -ERRORS_GENERIC = { - 1: "Informational", - 2: "Successful", - 3: "Redirection", - 4: "Client Error", - 5: "Server Error", - 7: "Developer Error" -} - -NETWORK_ERRORS = { - 100: "Continue", - 101: "Switching Protocols", - 200: "OK", - 201: "Created", - 202: "Accepted", - 203: "Non-Authoritative Information", - 204: "No Content", - 205: "Reset Content", - 206: "Partial Content", - 300: "Multiple Choices", - 301: "Moved Permanently", - 302: "Found", - 303: "See Other", - 304: "Not Modified", - 305: "Use Proxy", - 307: "Temporary Redirect", - 400: "Bad Request", - 401: "Unauthorized", - 402: "Payment Required", - 403: "Forbidden", - 404: "Not Found", - 405: "Method Not Allowed", - 406: "Not Acceptable", - 407: "Proxy Authentication Required", - 408: "Request Timeout", - 409: "Conflict", - 410: "Gone", - 411: "Length Required", - 412: "Precondition Failed", - 413: "Payload Too Large", - 414: "URI Too Long", - 415: "Unsupported Media Type", - 416: "Range Not Satisfiable", - 417: "Expectation Failed", - 418: "I'm a teapot", - 426: "Upgrade Required", - 500: "Internal Server Error", - 501: "Not Implemented", - 502: "Bad Gateway", - 503: "Service Unavailable", - 504: "Gateway Time-out", - 505: "HTTP Version Not Supported", - 102: "Processing", - 207: "Multi-Status", - 226: "IM Used", - 308: "Permanent Redirect", - 422: "Unprocessable Entity", - 423: "Locked", - 424: "Failed Dependency", - 428: "Precondition Required", - 429: "Too Many Requests", - 431: "Request Header Fields Too Large", - 451: "Unavailable For Legal Reasons", - 506: "Variant Also Negotiates", - 507: "Insufficient Storage", - 511: "Network Authentication Required" -} - -DEFAULT_CONFIG_FILE = os.path.join(os.path.expandvars(os.path.expanduser(__file__.split("openbao_cmds.py")[0])), 'openbao.env') - - -class OpenBao: - def __init__(self, conn:str, token:str): - self.conn = conn if conn is not None else '0.0.0.0:8200' - self.token = token - if self.token is None: - raise ValueError("Missing token value, cannot continue...") - - def __execute_get(self, url:str, headers:dict): - """ - Execute GET request - :args: - url:str - request URl - headers:dict - REST headers - :params: - response:requests.GET - response fromm GET - :return: - response - if response.status_code has a Client Error, return empty dict - """ - try: - response = requests.get(url=url, headers=headers) - except Exception as error: - raise Exception(f"Failed to execute GET against {url} (Error: {error}") - else: - if 400 <= response.status_code < 500: - response = {} - elif not 200 <= response.status_code < 300: - if int(response.status_code) in NETWORK_ERRORS: - raise ConnectionError(f"Failed to execute GET against {url} (Network Error {response.status_code}: {NETWORK_ERRORS[int(response.status_code)]})") - elif int(response.status_code[0]) in ERRORS_GENERIC: - raise ConnectionError(f"Failed to execute GET against {url} (Network Error {response.status_code}: {ERRORS_GENERIC[int(response.status_code[0])]}") - - return response - - - def get_openbao_status(self): - """ - health check for OpenBao - :anylog-cmd: - get openbao status where name=mybao - :args: - openbao_name:str - OpenBao connection name - :params: - headers:dict - REST headers - url:str - URL to access OpenBao - :return: - dictionary for OpenBao status - :sample output: - { - 'initialized': True, 'sealed': False, 'standby': False, 'performance_standby': False, - 'replication_performance_mode': 'disabled', 'replication_dr_mode': 'disabled', - 'server_time_utc': 1738794866, 'version': '2.1.1', 'cluster_name': 'vault-cluster-8acc91c5', - 'cluster_id': 'bd9670ca-b594-3c6b-6431-2d81afa9bdd0' - } - """ - headers = { - "X-Vault-Token": self.token, - "Content-Type": "application/json" - } - - url = f"http://{self.conn}/v1/sys/health" - response = self.__execute_get(url=url, headers=headers) - try: - return response.json() - except Exception as error: - raise Exception(f"Failed to extract results for {self.conn} (Error: {error})") - - - def get_openbao_value(self, section:str, key:str=None): - """ - Get secret value(s) within OpenBao - :anylog-cmd: - get openbao status where name=mybao and section=XXX [and key=XXX] - :args: - openbao_name:str - OpenBao connection name - section:str - subsection within secrets data key/value pairs will be stored - key:str - specific key to get value for - :params: - headers:dict - REST headers - url:str - URL to access OpenBao - :return: - if key - return specific value - if no key - return full dict of params - :note: - if section or key is invalid then returns error 404 - error should be "failed to get valid data" - """ - headers = { - "X-Vault-Token": self.token, - } - - url = f"http://{self.conn}/v1/secret/data/{section}" - response = self.__execute_get(url=url, headers=headers) - if response == {}: - return response - - try: - output = response.json() - except Exception as error: - raise Exception(f"Failed to extract key/values for {url} (Error: {error})") - - content = output['data']['data'][section] - if key: - if key in content: - return content[key] - else: - raise ValueError(f'Failed to find value for {key}') - else: - return content - - - def publish_data(self, section:str, payload:dict): - """ - Publish value to OpenBao - :args: - :args: - openbao_name:str - OpenBao connection name - section:str - subsection within secrets data key/value pairs will be stored - payload:dict - key/value pairs - :params: - headers:dict - REST headers - url:str - URL to access OpenBao - """ - headers = { - "X-Vault-Token": self.token, - "Content-Type": "application/json" - } - - orig_payloads = self.get_openbao_value(section=section) - updated_payloads = {**orig_payloads, **payload} - publish_payloads = {'data': { - section: updated_payloads - }} - - url = f"http://{self.conn}/v1/secret/data/{section}" - - try: - response = requests.post(url=url, headers=headers, json=publish_payloads) - except Exception as error: - raise Exception(f"Failed to POST data against {url} (Error: {error})") - else: - if not 200 <= response.status_code < 300: - if int(response.status_code) in NETWORK_ERRORS: - raise ConnectionError(f"Failed to execute POST against {url} (Network Error {response.status_code}: {NETWORK_ERRORS[int(response.status_code)]})") - elif int(response.status_code[0]) in ERRORS_GENERIC: - raise ConnectionError(f"Failed to execute POST against {url} (Network Error {response.status_code}: {ERRORS_GENERIC[int(response.status_code[0])]}") - - -def main(): - parse = argparse.ArgumentParser() - parse.add_argument('--config-file', type=str,default=DEFAULT_CONFIG_FILE, help='OpenBao .env config files') - parse.add_argument('--conn', type=str, default='0.0.0.0:8200', help='URL connection') - parse.add_argument('--token', type=str, default=None, help='OpenBao token') - parse.add_argument('--section', type=str, default='mybao', help='Section to secrets data location') - args = parse.parse_args() - - openbao = OpenBao(conn=args.conn, token=args.token) - args.config_file = os.path.expandvars(os.path.expanduser(args.config_file)) - if not os.path.isfile(args.config_file): - raise FileNotFoundError(f"Failed to locate config file {args.config_file}") - - # check status - print(openbao.get_openbao_status()) - configs = {} - with open(args.config_file, 'r') as f: - for line in f.read().split("\n"): - if not (line.startswith("#") or not line): - key, value = line.split("=") - try: - configs[key.strip()] = ast.literal_eval(value.strip()) - except: - configs[key.strip()] = value.strip() - - openbao.publish_data(section=args.section, payload=configs) - print(openbao.get_openbao_value(section=args.section)) - - -if __name__ == '__main__': - main() - # # 127.0.0.1:8200 --check-health - # openbao = OpenBao() - # openbao.connect_openbao(name='mybao', conn='0.0.0.0:8200', token='s.57hG4fLMZzRbbPF7mF5fA2Md') - # - # # check status - # print(openbao.get_openbao_status(openbao_name='mybao')) - # - # # create section and set payload value(s) - # openbao.publish_data(openba_name='mybao', section='anylog-data', payload={'param1': True, 'param2': 3, 'param3': 'abc'}) - # print(openbao.get_openbao_value(openbao_name='mybao', section='anylog-data')) - # - # # add another value and update value to payload - # openbao.publish_data(openba_name='mybao', section='anylog-data', payload={'param1': False, 'param4': ''}) - # print(openbao.get_openbao_value(openbao_name='mybao', section='anylog-data')) \ No newline at end of file diff --git a/openbao/run_openbao.sh b/openbao/run_openbao.sh deleted file mode 100644 index 9caa0d63..00000000 --- a/openbao/run_openbao.sh +++ /dev/null @@ -1,7 +0,0 @@ -docker run -d \ - --name openbao \ - -p 8080:8080 \ - -p 8200:8200 \ - -v openbao-config:/config \ - -v openbao-data:/data \ ---rm openbao/openbao:latest \ No newline at end of file diff --git a/row_count.sh b/row_count.sh deleted file mode 100644 index 7234c067..00000000 --- a/row_count.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -NODE_CONN=${1:-127.0.0.1:32349} -DB_NAME=${2:-mybd} -TIMESTAMP=${3:-insert_timestamp} - -# check status - fails do not continue -curl -X GET "http://${NODE_CONN}" -w "\n" - -TABLES=$(curl -s -X GET "http://${NODE_CONN}" \ - -H "command: blockchain get table where dbms=${DB_NAME} bring [*][name] separator=\n" \ - -H "User-Agent: AnyLog/1.23" -) - -echo "Current Timestamp (UTC): $(date -u +"%Y-%m-%d %H:%M:%S")" - -for TABLE in ${TABLES}; do - echo "${TABLE}" - - for WHERE_CONDITION in \ - "${TIMESTAMP} >= NOW() - 1 hour" \ - "period(hour, 1, now(), ${TIMESTAMP})" - do - echo "${TABLE} - ${WHERE_CONDITION}" - - curl -s -X GET "http://${NODE_CONN}" \ - -H "command: sql ${DB_NAME} format=table and timezone=utc and extend=(+node_name) \ -SELECT MIN(${TIMESTAMP}) as min_ts, MAX(${TIMESTAMP}), COUNT(*) FROM ${TABLE} WHERE ${WHERE_CONDITION}" \ - -H "User-Agent: AnyLog/1.23" \ - -H "destination: network" \ - -w "\n" - done -done \ No newline at end of file From d86149aba8ffaa8cbdead45b2034dcf0d94381d7 Mon Sep 17 00:00:00 2001 From: Ori Shadmon Date: Wed, 13 May 2026 14:41:23 -0700 Subject: [PATCH 29/46] revert os-dev for subset of code Signed-off-by: Ori Shadmon --- customers/dados.proto | 19 ++ customers/dados_pb2.py | 42 +++ customers/dados_pb2_grpc.py | 97 +++++++ customers/grpc_client.al | 14 + .../anylog-generic/node_configs.env | 2 +- .../anylog-master/node_configs.env | 2 +- .../anylog-operator/node_configs.env | 54 ++-- .../anylog-publisher/node_configs.env | 2 +- .../anylog-query/node_configs.env | 2 +- .../node_configs.env | 107 ++++--- .../node_configs.env | 2 +- docker-makefiles/prep_configs.sh | 5 +- openbao/docker-compose.yml | 24 ++ openbao/get4bao.sh | 6 + openbao/openbao.env | 9 + openbao/openbao_cmds.py | 270 ++++++++++++++++++ openbao/run_openbao.sh | 7 + 17 files changed, 589 insertions(+), 75 deletions(-) create mode 100644 customers/dados.proto create mode 100644 customers/dados_pb2.py create mode 100644 customers/dados_pb2_grpc.py create mode 100644 customers/grpc_client.al create mode 100644 openbao/docker-compose.yml create mode 100644 openbao/get4bao.sh create mode 100644 openbao/openbao.env create mode 100644 openbao/openbao_cmds.py create mode 100644 openbao/run_openbao.sh diff --git a/customers/dados.proto b/customers/dados.proto new file mode 100644 index 00000000..d05dbaf5 --- /dev/null +++ b/customers/dados.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; +package persistence; + +service PersistenceService { + rpc WatchTable (WatchRequest) returns (stream DataRow); +} + +message WatchRequest { + string table_id = 1; +} + +message DataRow { + string data = 1; +} + +// missing reply function +message ReplyMessage { + int32 Retval = 1; +} diff --git a/customers/dados_pb2.py b/customers/dados_pb2.py new file mode 100644 index 00000000..1f41d821 --- /dev/null +++ b/customers/dados_pb2.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: dados.proto +# Protobuf Python Version: 6.31.1 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 31, + 1, + '', + 'dados.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0b\x64\x61\x64os.proto\x12\x0bpersistence\" \n\x0cWatchRequest\x12\x10\n\x08table_id\x18\x01 \x01(\t\"\x17\n\x07\x44\x61taRow\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\t\"\x1e\n\x0cReplyMessage\x12\x0e\n\x06Retval\x18\x01 \x01(\x05\x32U\n\x12PersistenceService\x12?\n\nWatchTable\x12\x19.persistence.WatchRequest\x1a\x14.persistence.DataRow0\x01\x62\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'dados_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + DESCRIPTOR._loaded_options = None + _globals['_WATCHREQUEST']._serialized_start=28 + _globals['_WATCHREQUEST']._serialized_end=60 + _globals['_DATAROW']._serialized_start=62 + _globals['_DATAROW']._serialized_end=85 + _globals['_REPLYMESSAGE']._serialized_start=87 + _globals['_REPLYMESSAGE']._serialized_end=117 + _globals['_PERSISTENCESERVICE']._serialized_start=119 + _globals['_PERSISTENCESERVICE']._serialized_end=204 +# @@protoc_insertion_point(module_scope) diff --git a/customers/dados_pb2_grpc.py b/customers/dados_pb2_grpc.py new file mode 100644 index 00000000..d93c3bf0 --- /dev/null +++ b/customers/dados_pb2_grpc.py @@ -0,0 +1,97 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc +import warnings + +import dados_pb2 as dados__pb2 + +GRPC_GENERATED_VERSION = '1.76.0' +GRPC_VERSION = grpc.__version__ +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + raise RuntimeError( + f'The grpc package installed is at version {GRPC_VERSION},' + + ' but the generated code in dados_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + ) + + +class PersistenceServiceStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.WatchTable = channel.unary_stream( + '/persistence.PersistenceService/WatchTable', + request_serializer=dados__pb2.WatchRequest.SerializeToString, + response_deserializer=dados__pb2.DataRow.FromString, + _registered_method=True) + + +class PersistenceServiceServicer(object): + """Missing associated documentation comment in .proto file.""" + + def WatchTable(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_PersistenceServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'WatchTable': grpc.unary_stream_rpc_method_handler( + servicer.WatchTable, + request_deserializer=dados__pb2.WatchRequest.FromString, + response_serializer=dados__pb2.DataRow.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'persistence.PersistenceService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + server.add_registered_method_handlers('persistence.PersistenceService', rpc_method_handlers) + + + # This class is part of an EXPERIMENTAL API. +class PersistenceService(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def WatchTable(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream( + request, + target, + '/persistence.PersistenceService/WatchTable', + dados__pb2.WatchRequest.SerializeToString, + dados__pb2.DataRow.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/customers/grpc_client.al b/customers/grpc_client.al new file mode 100644 index 00000000..0b26a44b --- /dev/null +++ b/customers/grpc_client.al @@ -0,0 +1,14 @@ + \ No newline at end of file diff --git a/docker-makefiles/anylog-generic/node_configs.env b/docker-makefiles/anylog-generic/node_configs.env index 87b68ed1..2af268ca 100644 --- a/docker-makefiles/anylog-generic/node_configs.env +++ b/docker-makefiles/anylog-generic/node_configs.env @@ -282,7 +282,7 @@ BUCKET_SECRET_KEY="" # Whether to enable Remote-GUI in deployment ENABLE_REMOTE_GUI=false # Remote-GUI version Tag -REMOTE_GUI_TAG=1.1.0 +REMOTE_GUI_TAG=beta2 # NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 REMOTE_GUI_NIC="" # Remote GUI Frontend Port diff --git a/docker-makefiles/anylog-master/node_configs.env b/docker-makefiles/anylog-master/node_configs.env index 86c7f1e3..f5dd3bcb 100644 --- a/docker-makefiles/anylog-master/node_configs.env +++ b/docker-makefiles/anylog-master/node_configs.env @@ -281,7 +281,7 @@ BUCKET_SECRET_KEY="" # Whether to enable Remote-GUI in deployment ENABLE_REMOTE_GUI=false # Remote-GUI version Tag -REMOTE_GUI_TAG=1.1.0 +REMOTE_GUI_TAG=beta2 # NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 REMOTE_GUI_NIC="" # Remote GUI Frontend Port diff --git a/docker-makefiles/anylog-operator/node_configs.env b/docker-makefiles/anylog-operator/node_configs.env index 730b6054..1edc8566 100644 --- a/docker-makefiles/anylog-operator/node_configs.env +++ b/docker-makefiles/anylog-operator/node_configs.env @@ -14,10 +14,10 @@ IMAGE=anylogco/anylog-network # the repo if needed OR a local path for deployment-scripts and then use that instead of cloning into the repositroy DEPLOYMENTS_REPO=https://github.com/AnyLog-co/deployment-scripts # Branch associated with git repo -DEPLOYMENTS_BRANCH=pre-develop +DEPLOYMENTS_BRANCH="pre-develop" # specify the docker socket - to be used when deploying docker monitoring -DOCKER_SOCKET=/var/run/docker.sock +DOCKER_SOCKET="/var/run/docker.sock" #==== basic ==== @@ -147,9 +147,9 @@ MSG_VALUE_COLUMN_TYPE=float # Whether or not to enable to OPC-UA service ENABLE_OPCUA=false # OPC-UA URL address (ex. opcua.tcp:;//127.0.0.1:4840) -OPCUA_URL= +OPCUA_URL="" # Node information the root is located in (ex. ns=2;s=DataSet) -OPCUA_NODE= +OPCUA_NODE="" # How often to pull data from OPC-UA OPCUA_FREQUENCY=5 @@ -160,8 +160,8 @@ OPCUA_FREQUENCY=5 ANYLOG_PATH=/app # !local_scripts: ${ANYLOG_HOME}/deployment-scripts/scripts LOCAL_SCRIPTS=/app/deployment-scripts -# !test_dir: writable test output directory -TEST_DIR=/app/AnyLog-Network/data/test +# !test_dir: ${ANYLOG_HOME}/deployment-scripts/test +TEST_DIR=/app/deployment-scripts/test #--- General --- # hide blockchain policy - supports all node types except cluster / operator @@ -169,29 +169,29 @@ IS_HIDDEN=false # Whether to automatically run a local (or personalized) script at the end of the process DEPLOY_LOCAL_SCRIPT=false # Coordinates of the machine - used by Grafana to map the network -LOCATION= +LOCATION="" # Country where machine is located -COUNTRY= +COUNTRY="" # State where machine is located -STATE= +STATE="" # City where machine is located -CITY= +CITY="" #--- Networking --- # Declare Policy name -CONFIG_NAME= +CONFIG_NAME="" # Whether to enable DNS resolution ENABLE_DNS=false # NIC type used to resolve the internal IP address (ex. eth0, lo) -NIC_TYPE= +NIC_TYPE="" # Overlay IP address - if set, will replace local IP address when connecting to network -OVERLAY_IP= +OVERLAY_IP="" # External DNS resolution -EXTERNAL_DNS= +EXTERNAL_DNS="" # local DNS domain to append to hostname (ex. example.com) -DNS_DOMAIN= +DNS_DOMAIN="" # Full path for machine's (local) DNS -DNS= +DNS="" # Bind settings - a true value binds to a specific IP and Port (a false value binds to all IPs) TCP_BIND=true @@ -214,20 +214,20 @@ BLOBS_COMPRESS=true # Folder for storing blobs locally # Leave empty to use default application path -BLOBS_FOLDER= +BLOBS_FOLDER="" #--- Object Store --- # Logical bucket grouping -BUCKET_GROUP= +BUCKET_GROUP="" # Bucket identifier -BUCKET_ID= +BUCKET_ID="" # Cloud region -BUCKET_REGION= +BUCKET_REGION="" #--- Operator --- # Operator ID - autogenerated INT if not specified # This is used to help keep a record of where data was originally stored -MEMBER= +MEMBER="" # Whether of not to enable HA against the cluster ENABLE_HA=false # How many days back to sync between nodes @@ -253,7 +253,7 @@ QUERY_POOL=6 #--- General --- # AnyLog License Key -LICENSE_KEY= +LICENSE_KEY="" #--- Database --- # Username for SQL database connection @@ -274,22 +274,22 @@ MQTT_PASSWD=mqtt4AnyLog! #--- Object Store --- # Access credentials -BUCKET_ACCESS_KEY= -BUCKET_SECRET_KEY= +BUCKET_ACCESS_KEY="" +BUCKET_SECRET_KEY="" #==== remote-gui ==== # Companion UI application - deployed alongside the AnyLog node # Whether to enable Remote-GUI in deployment ENABLE_REMOTE_GUI=false # Remote-GUI version Tag -REMOTE_GUI_TAG=1.1.0 +REMOTE_GUI_TAG=beta2 # NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 -REMOTE_GUI_NIC= +REMOTE_GUI_NIC="" # Remote GUI Frontend Port REMOTE_GUI_FE=31800 # Remote GUI Backend Port REMOTE_GUI_BE=8080 # Grafana PATH to reference for Grafana connection -GRAFANA_URL= +GRAFANA_URL="" diff --git a/docker-makefiles/anylog-publisher/node_configs.env b/docker-makefiles/anylog-publisher/node_configs.env index 69971810..e49a862b 100644 --- a/docker-makefiles/anylog-publisher/node_configs.env +++ b/docker-makefiles/anylog-publisher/node_configs.env @@ -282,7 +282,7 @@ BUCKET_SECRET_KEY="" # Whether to enable Remote-GUI in deployment ENABLE_REMOTE_GUI=false # Remote-GUI version Tag -REMOTE_GUI_TAG=1.1.0 +REMOTE_GUI_TAG=beta2 # NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 REMOTE_GUI_NIC="" # Remote GUI Frontend Port diff --git a/docker-makefiles/anylog-query/node_configs.env b/docker-makefiles/anylog-query/node_configs.env index 305e80c1..fc7bc875 100644 --- a/docker-makefiles/anylog-query/node_configs.env +++ b/docker-makefiles/anylog-query/node_configs.env @@ -282,7 +282,7 @@ BUCKET_SECRET_KEY="" # Whether to enable Remote-GUI in deployment ENABLE_REMOTE_GUI=true # Remote-GUI version Tag -REMOTE_GUI_TAG=1.1.0 +REMOTE_GUI_TAG=beta2 # NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 REMOTE_GUI_NIC="" # Remote GUI Frontend Port diff --git a/docker-makefiles/anylog-standalone-operator/node_configs.env b/docker-makefiles/anylog-standalone-operator/node_configs.env index 05cc8fd4..ebb760b7 100644 --- a/docker-makefiles/anylog-standalone-operator/node_configs.env +++ b/docker-makefiles/anylog-standalone-operator/node_configs.env @@ -12,18 +12,38 @@ IMAGE=anylogco/anylog-network # Github repo to use for configuring / deploying node # The path can either be a Github URL and then the code will automatically clone # the repo if needed OR a local path for deployment-scripts and then use that instead of cloning into the repositroy -#DEPLOYMENTS_REPO=https://github.com/AnyLog-co/deployment-scripts -DEPLOYMENTS_REPO=/Users/mdavidson58/Documents/Anylog/pre-develop/deployment-scripts # Branch associated with git repo -DEPLOYMENTS_BRANCH=os-dev +DEPLOYMENTS_REPO="/mnt/c/Users/oshad/AnyLog-code/deployment-scripts" +DEPLOYMENTS_BRANCH="os-dev" # specify the docker socket - to be used when deploying docker monitoring -DOCKER_SOCKET=/var/run/docker.sock +DOCKER_SOCKET="/var/run/docker.sock" + +# Local folders to mount inside the AnyLog container. +# +# Examples: +# No volumes: +# USER_VOLUMES="" +# +# Single volume: +# USER_VOLUMES="/home/user/grpc-scripts" +# +# Multiple volumes (space-separated): +# USER_VOLUMES="/home/user/grpc-scripts /home/user/local-deployments" +# +# Notes: +# - Use absolute paths only +# - Paths are mounted automatically under /app/ +# +# Example: +# /home/user/grpc-scripts -> /app/grpc-scripts +# +USER_VOLUMES="/mnt/c/Users/oshad/AnyLog-code/docker-compose/customers" #==== basic ==== # Core node configuration - required for all deployments #--- General --- -# Configuration based deploymnet node deployment type +# Configuration based deployment node deployment type # Node Types: # generic: sandbox with only network configured # master: "blockchain" emulator @@ -89,20 +109,24 @@ BLOCKCHAIN_DESTINATION=file # Owner of the cluster CLUSTER_NAME=anylog-cluster1 # Logical database name -DEFAULT_DBMS=new_company +DEFAULT_DBMS=persistence #==== southbound ==== # Data ingestion protocols (MQTT, OPC-UA, video streaming, monitoring) + #--- Monitoring --- # Whether to monitor the node or not -NODE_MONITORING=false +NODE_MONITORING=true # Store monitoring in Operator node(s) -STORE_MONITORING=false +STORE_MONITORING=true # Accept syslog data from local (Message broker required) SYSLOG_MONITORING=true # Docker Monitoring DOCKER_MONITORING=true +# how often data is being pulled from docker +DOCKER_FREQUENCY = 30 + #--- Video Streaming --- # Enable or disable video streaming. # true = stream video, false = do not stream @@ -121,7 +145,7 @@ YOLO_MODEL_PORT=50051 #--- MQTT --- # Whether to enable the default MQTT process -ENABLE_MQTT=true +ENABLE_MQTT=false # IP address of MQTT broker MQTT_BROKER=172.104.228.251 @@ -149,7 +173,7 @@ ENABLE_OPCUA=true # OPC-UA URL address (ex. opcua.tcp:;//127.0.0.1:4840) OPCUA_URL=opc.tcp://virtualfactory.proveit.services:4842/discovery # Node information the root is located in (ex. ns=2;s=DataSet) -OPCUA_NODE=ns=2;s=Site1 +OPCUA_NODE="ns=2;s=Site1" # How often to pull data from OPC-UA OPCUA_FREQUENCY=5 @@ -160,41 +184,41 @@ OPCUA_FREQUENCY=5 ANYLOG_PATH=/app # !local_scripts: ${ANYLOG_HOME}/deployment-scripts/scripts LOCAL_SCRIPTS=/app/deployment-scripts -# !test_dir: writable test output directory -TEST_DIR=/app/AnyLog-Network/data/test +# !test_dir: ${ANYLOG_HOME}/deployment-scripts/test +TEST_DIR=/app/deployment-scripts/test #--- General --- # hide blockchain policy - supports all node types except cluster / operator IS_HIDDEN=false # Whether to automatically run a local (or personalized) script at the end of the process -DEPLOY_LOCAL_SCRIPT=true +DEPLOY_LOCAL_SCRIPT=false # Coordinates of the machine - used by Grafana to map the network -LOCATION= +LOCATION="" # Country where machine is located -COUNTRY= +COUNTRY="" # State where machine is located -STATE= +STATE="" # City where machine is located -CITY= +CITY="" #--- Networking --- # Declare Policy name -CONFIG_NAME= +CONFIG_NAME="" # Whether to enable DNS resolution ENABLE_DNS=false # NIC type used to resolve the internal IP address (ex. eth0, lo) -NIC_TYPE= +NIC_TYPE="" # Overlay IP address - if set, will replace local IP address when connecting to network -OVERLAY_IP= +OVERLAY_IP="" # External DNS resolution -EXTERNAL_DNS= +EXTERNAL_DNS="" # local DNS domain to append to hostname (ex. example.com) -DNS_DOMAIN= +DNS_DOMAIN="" # Full path for machine's (local) DNS -DNS= +DNS="" # Bind settings - a true value binds to a specific IP and Port (a false value binds to all IPs) -TCP_BIND=true +TCP_BIND=false REST_BIND=false BROKER_BIND=false @@ -203,7 +227,8 @@ TCP_THREADS=6 REST_THREADS=6 BROKER_THREADS=6 -# ST_TIMEOUT=30 +# Timeout in seconds for REST requests (0 = continuous) +REST_TIMEOUT=30 #--- NoSQL --- # Reuse identical blobs using file hash @@ -213,38 +238,40 @@ BLOBS_COMPRESS=true # Folder for storing blobs locally # Leave empty to use default application path -BLOBS_FOLDER= +BLOBS_FOLDER="" #--- Object Store --- # Logical bucket grouping -BUCKET_GROUP= +BUCKET_GROUP="" # Bucket identifier -BUCKET_ID= +BUCKET_ID="" # Cloud region -BUCKET_REGION= +BUCKET_REGION="" #--- Operator --- # Operator ID - autogenerated INT if not specified # This is used to help keep a record of where data was originally stored -MEMBER= +MEMBER="" # Whether of not to enable HA against the cluster ENABLE_HA=false # How many days back to sync between nodes START_DATE=30 # How many threads to use in the operator process OPERATOR_THREADS=3 - +# Parallel threads to be used in order digest data into AnyLog +OPERATOR_HELPERS=5 #--- Sub processes --- # Compress JSON and SQL file(s) backup COMPRESS_FILE=true # When data comes in write to database immediately, as opposed to waiting for a full buffer WRITE_IMMEDIATE=true # If buffer is not full, how long to wait until pushing data through -THRESHOLD_TIME=60 seconds +THRESHOLD_TIME=10 seconds # Buffer size to reach, at which point data is pushed through -THRESHOLD_VOLUME=10KB +THRESHOLD_VOLUME=1KB # Number of parallel queries QUERY_POOL=6 +# for `run operator` process, specify the number of parallel threads to use #==== secrets ==== # Sensitive credentials - do not commit to version control @@ -273,23 +300,23 @@ MQTT_PASSWD=mqtt4AnyLog! #--- Object Store --- # Access credentials -BUCKET_ACCESS_KEY= -BUCKET_SECRET_KEY= +BUCKET_ACCESS_KEY="" +BUCKET_SECRET_KEY="" #==== remote-gui ==== # Companion UI application - deployed alongside the AnyLog node # Whether to enable Remote-GUI in deployment -ENABLE_REMOTE_GUI=true +ENABLE_REMOTE_GUI=false # Remote-GUI version Tag -REMOTE_GUI_TAG=1.1.0 +REMOTE_GUI_TAG=beta2 # NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 -REMOTE_GUI_NIC= +REMOTE_GUI_NIC="" # Remote GUI Frontend Port REMOTE_GUI_FE=31800 # Remote GUI Backend Port REMOTE_GUI_BE=8080 # Default connection to REST service (IP:Port) if not provided then autogenerated using `REMOTE_GUI_NIC` -REMOTE_CONN= +REMOTE_CONN="" # Grafana PATH to reference for Grafana connection -GRAFANA_URL= +GRAFANA_URL="" diff --git a/docker-makefiles/anylog-standalone-publisher/node_configs.env b/docker-makefiles/anylog-standalone-publisher/node_configs.env index 20c673e8..78003efb 100644 --- a/docker-makefiles/anylog-standalone-publisher/node_configs.env +++ b/docker-makefiles/anylog-standalone-publisher/node_configs.env @@ -281,7 +281,7 @@ BUCKET_SECRET_KEY="" # Whether to enable Remote-GUI in deployment ENABLE_REMOTE_GUI=true # Remote-GUI version Tag -REMOTE_GUI_TAG=1.1.0 +REMOTE_GUI_TAG=beta2 # NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 REMOTE_GUI_NIC="" # Remote GUI Frontend Port diff --git a/docker-makefiles/prep_configs.sh b/docker-makefiles/prep_configs.sh index f1c230ac..c3c4f30a 100644 --- a/docker-makefiles/prep_configs.sh +++ b/docker-makefiles/prep_configs.sh @@ -47,9 +47,8 @@ fi normalize_quotes() { local file="$1" echo "Normalizing quotes in: ${file}" - sedi "s/=''$/=/g" "${file}" - sedi -E "s/^([A-Za-z_][A-Za-z0-9_]*)='(.*)'/\1=\2/" "${file}" - sedi -E "s/^([A-Za-z_][A-Za-z0-9_]*)=\"(.*)\"/\1=\2/" "${file}" + sedi "s/=''/=\"\"/g" "${file}" + sedi -E "s/^([A-Za-z_][A-Za-z0-9_]*)='(.*)'/\1=\"\2\"/" "${file}" } for cfg in "${CONFIG_FILES[@]}"; do diff --git a/openbao/docker-compose.yml b/openbao/docker-compose.yml new file mode 100644 index 00000000..5b195380 --- /dev/null +++ b/openbao/docker-compose.yml @@ -0,0 +1,24 @@ +services: + anylog-node: + image: anylogco/anylog-network:1.3.2412-beta8 + restart: always + container_name: anylog-node + stdin_open: true + tty: true + network_mode: host + environment: + - INIT_TYPE=prod + - OPENBAO_VAULT_IP=http://0.0.0.0:8200 + - OPENBAO_VAULT_TOKEN=s.57hG4fLMZzRbbPF7mF5fA2Md + - SECTION_NAME=mybao + volumes: + - anylog-node-anylog:/app/AnyLog-Network/anylog + - anylog-node-blockchain:/app/AnyLog-Network/blockchain + - anylog-node-data:/app/AnyLog-Network/data + - anylog-node-local-scripts:/app/deployment-scripts + +volumes: + anylog-node-anylog: + anylog-node-blockchain: + anylog-node-data: + anylog-node-local-scripts: \ No newline at end of file diff --git a/openbao/get4bao.sh b/openbao/get4bao.sh new file mode 100644 index 00000000..9a2629b8 --- /dev/null +++ b/openbao/get4bao.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# 0.0.0.0:8200 s.Pb8yAsBHAKyva0DHmQA4Ib12 +python3 get4bao.py ${VAULT_URL} ${VAULT_TOKEN} + +source $ANYLOG_PATH/openbao/configs.env \ No newline at end of file diff --git a/openbao/openbao.env b/openbao/openbao.env new file mode 100644 index 00000000..9bd4ed5f --- /dev/null +++ b/openbao/openbao.env @@ -0,0 +1,9 @@ +# test configs +SERVICE_PORT=32548 +TCP_BIND=true +TCP_THREADS=3 + +REST_PORT=32549 +REST_TIMEOUT=30 +REST_BIND=false +REST_THREADS=3 \ No newline at end of file diff --git a/openbao/openbao_cmds.py b/openbao/openbao_cmds.py new file mode 100644 index 00000000..3b246854 --- /dev/null +++ b/openbao/openbao_cmds.py @@ -0,0 +1,270 @@ +# URL: https://openbao.org/api-docs/ +import argparse +import ast +import json +import os +import requests + +ERRORS_GENERIC = { + 1: "Informational", + 2: "Successful", + 3: "Redirection", + 4: "Client Error", + 5: "Server Error", + 7: "Developer Error" +} + +NETWORK_ERRORS = { + 100: "Continue", + 101: "Switching Protocols", + 200: "OK", + 201: "Created", + 202: "Accepted", + 203: "Non-Authoritative Information", + 204: "No Content", + 205: "Reset Content", + 206: "Partial Content", + 300: "Multiple Choices", + 301: "Moved Permanently", + 302: "Found", + 303: "See Other", + 304: "Not Modified", + 305: "Use Proxy", + 307: "Temporary Redirect", + 400: "Bad Request", + 401: "Unauthorized", + 402: "Payment Required", + 403: "Forbidden", + 404: "Not Found", + 405: "Method Not Allowed", + 406: "Not Acceptable", + 407: "Proxy Authentication Required", + 408: "Request Timeout", + 409: "Conflict", + 410: "Gone", + 411: "Length Required", + 412: "Precondition Failed", + 413: "Payload Too Large", + 414: "URI Too Long", + 415: "Unsupported Media Type", + 416: "Range Not Satisfiable", + 417: "Expectation Failed", + 418: "I'm a teapot", + 426: "Upgrade Required", + 500: "Internal Server Error", + 501: "Not Implemented", + 502: "Bad Gateway", + 503: "Service Unavailable", + 504: "Gateway Time-out", + 505: "HTTP Version Not Supported", + 102: "Processing", + 207: "Multi-Status", + 226: "IM Used", + 308: "Permanent Redirect", + 422: "Unprocessable Entity", + 423: "Locked", + 424: "Failed Dependency", + 428: "Precondition Required", + 429: "Too Many Requests", + 431: "Request Header Fields Too Large", + 451: "Unavailable For Legal Reasons", + 506: "Variant Also Negotiates", + 507: "Insufficient Storage", + 511: "Network Authentication Required" +} + +DEFAULT_CONFIG_FILE = os.path.join(os.path.expandvars(os.path.expanduser(__file__.split("openbao_cmds.py")[0])), 'openbao.env') + + +class OpenBao: + def __init__(self, conn:str, token:str): + self.conn = conn if conn is not None else '0.0.0.0:8200' + self.token = token + if self.token is None: + raise ValueError("Missing token value, cannot continue...") + + def __execute_get(self, url:str, headers:dict): + """ + Execute GET request + :args: + url:str - request URl + headers:dict - REST headers + :params: + response:requests.GET - response fromm GET + :return: + response + if response.status_code has a Client Error, return empty dict + """ + try: + response = requests.get(url=url, headers=headers) + except Exception as error: + raise Exception(f"Failed to execute GET against {url} (Error: {error}") + else: + if 400 <= response.status_code < 500: + response = {} + elif not 200 <= response.status_code < 300: + if int(response.status_code) in NETWORK_ERRORS: + raise ConnectionError(f"Failed to execute GET against {url} (Network Error {response.status_code}: {NETWORK_ERRORS[int(response.status_code)]})") + elif int(response.status_code[0]) in ERRORS_GENERIC: + raise ConnectionError(f"Failed to execute GET against {url} (Network Error {response.status_code}: {ERRORS_GENERIC[int(response.status_code[0])]}") + + return response + + + def get_openbao_status(self): + """ + health check for OpenBao + :anylog-cmd: + get openbao status where name=mybao + :args: + openbao_name:str - OpenBao connection name + :params: + headers:dict - REST headers + url:str - URL to access OpenBao + :return: + dictionary for OpenBao status + :sample output: + { + 'initialized': True, 'sealed': False, 'standby': False, 'performance_standby': False, + 'replication_performance_mode': 'disabled', 'replication_dr_mode': 'disabled', + 'server_time_utc': 1738794866, 'version': '2.1.1', 'cluster_name': 'vault-cluster-8acc91c5', + 'cluster_id': 'bd9670ca-b594-3c6b-6431-2d81afa9bdd0' + } + """ + headers = { + "X-Vault-Token": self.token, + "Content-Type": "application/json" + } + + url = f"http://{self.conn}/v1/sys/health" + response = self.__execute_get(url=url, headers=headers) + try: + return response.json() + except Exception as error: + raise Exception(f"Failed to extract results for {self.conn} (Error: {error})") + + + def get_openbao_value(self, section:str, key:str=None): + """ + Get secret value(s) within OpenBao + :anylog-cmd: + get openbao status where name=mybao and section=XXX [and key=XXX] + :args: + openbao_name:str - OpenBao connection name + section:str - subsection within secrets data key/value pairs will be stored + key:str - specific key to get value for + :params: + headers:dict - REST headers + url:str - URL to access OpenBao + :return: + if key - return specific value + if no key - return full dict of params + :note: + if section or key is invalid then returns error 404 - error should be "failed to get valid data" + """ + headers = { + "X-Vault-Token": self.token, + } + + url = f"http://{self.conn}/v1/secret/data/{section}" + response = self.__execute_get(url=url, headers=headers) + if response == {}: + return response + + try: + output = response.json() + except Exception as error: + raise Exception(f"Failed to extract key/values for {url} (Error: {error})") + + content = output['data']['data'][section] + if key: + if key in content: + return content[key] + else: + raise ValueError(f'Failed to find value for {key}') + else: + return content + + + def publish_data(self, section:str, payload:dict): + """ + Publish value to OpenBao + :args: + :args: + openbao_name:str - OpenBao connection name + section:str - subsection within secrets data key/value pairs will be stored + payload:dict - key/value pairs + :params: + headers:dict - REST headers + url:str - URL to access OpenBao + """ + headers = { + "X-Vault-Token": self.token, + "Content-Type": "application/json" + } + + orig_payloads = self.get_openbao_value(section=section) + updated_payloads = {**orig_payloads, **payload} + publish_payloads = {'data': { + section: updated_payloads + }} + + url = f"http://{self.conn}/v1/secret/data/{section}" + + try: + response = requests.post(url=url, headers=headers, json=publish_payloads) + except Exception as error: + raise Exception(f"Failed to POST data against {url} (Error: {error})") + else: + if not 200 <= response.status_code < 300: + if int(response.status_code) in NETWORK_ERRORS: + raise ConnectionError(f"Failed to execute POST against {url} (Network Error {response.status_code}: {NETWORK_ERRORS[int(response.status_code)]})") + elif int(response.status_code[0]) in ERRORS_GENERIC: + raise ConnectionError(f"Failed to execute POST against {url} (Network Error {response.status_code}: {ERRORS_GENERIC[int(response.status_code[0])]}") + + +def main(): + parse = argparse.ArgumentParser() + parse.add_argument('--config-file', type=str,default=DEFAULT_CONFIG_FILE, help='OpenBao .env config files') + parse.add_argument('--conn', type=str, default='0.0.0.0:8200', help='URL connection') + parse.add_argument('--token', type=str, default=None, help='OpenBao token') + parse.add_argument('--section', type=str, default='mybao', help='Section to secrets data location') + args = parse.parse_args() + + openbao = OpenBao(conn=args.conn, token=args.token) + args.config_file = os.path.expandvars(os.path.expanduser(args.config_file)) + if not os.path.isfile(args.config_file): + raise FileNotFoundError(f"Failed to locate config file {args.config_file}") + + # check status + print(openbao.get_openbao_status()) + configs = {} + with open(args.config_file, 'r') as f: + for line in f.read().split("\n"): + if not (line.startswith("#") or not line): + key, value = line.split("=") + try: + configs[key.strip()] = ast.literal_eval(value.strip()) + except: + configs[key.strip()] = value.strip() + + openbao.publish_data(section=args.section, payload=configs) + print(openbao.get_openbao_value(section=args.section)) + + +if __name__ == '__main__': + main() + # # 127.0.0.1:8200 --check-health + # openbao = OpenBao() + # openbao.connect_openbao(name='mybao', conn='0.0.0.0:8200', token='s.57hG4fLMZzRbbPF7mF5fA2Md') + # + # # check status + # print(openbao.get_openbao_status(openbao_name='mybao')) + # + # # create section and set payload value(s) + # openbao.publish_data(openba_name='mybao', section='anylog-data', payload={'param1': True, 'param2': 3, 'param3': 'abc'}) + # print(openbao.get_openbao_value(openbao_name='mybao', section='anylog-data')) + # + # # add another value and update value to payload + # openbao.publish_data(openba_name='mybao', section='anylog-data', payload={'param1': False, 'param4': ''}) + # print(openbao.get_openbao_value(openbao_name='mybao', section='anylog-data')) \ No newline at end of file diff --git a/openbao/run_openbao.sh b/openbao/run_openbao.sh new file mode 100644 index 00000000..9caa0d63 --- /dev/null +++ b/openbao/run_openbao.sh @@ -0,0 +1,7 @@ +docker run -d \ + --name openbao \ + -p 8080:8080 \ + -p 8200:8200 \ + -v openbao-config:/config \ + -v openbao-data:/data \ +--rm openbao/openbao:latest \ No newline at end of file From 3a26507ffa9dd803484157bf0e19db05fad91e24 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 13 May 2026 21:41:39 +0000 Subject: [PATCH 30/46] chore: update version for os-dev [skip ci] --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 4f692a41..182e81e9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = docker-compose -version = 1.4.2604 (os-dev - d6daac [2026-05-13 03:05:52]) +version = 1.4.2604 (os-dev - d86149 [2026-05-13 21:41:39]) author = AnyLog Co. contact = info@anylog.co description = docker-compose configurations for AnyLog deployment From a7d90875bf7fb37378c7dd58b130200f032edc4a Mon Sep 17 00:00:00 2001 From: Ori Shadmon Date: Wed, 13 May 2026 15:49:26 -0700 Subject: [PATCH 31/46] if license came from file ignore license process Signed-off-by: Ori Shadmon --- deploy.sh | 53 ++++++++++--------- .../node_configs.env | 2 +- docker-makefiles/build_docker_compose.sh | 2 +- docker-makefiles/prep_configs.sh | 2 +- license-generator/license_key.sh | 1 + 5 files changed, 32 insertions(+), 28 deletions(-) diff --git a/deploy.sh b/deploy.sh index 8835ec31..f1848483 100644 --- a/deploy.sh +++ b/deploy.sh @@ -2,7 +2,7 @@ # deploy.sh — AnyLog node lifecycle manager # Usage: bash deploy.sh [OPTIONS] # Run: bash deploy.sh help -#set -euo pipefail +set -euo pipefail # ────────────────────────────────────────────── # Defaults (override via environment or flags) @@ -302,39 +302,33 @@ cmd_license_check() { local env_file="docker-makefiles/${ANYLOG_TYPE}/.env" local single_file="docker-makefiles/${ANYLOG_TYPE}/node_configs.env" local license_arg=() - local accepted_license="" + local license_from_file=false case "${PROMPT_LICENSE}" in true|True|TRUE|1|yes|Yes|YES) PROMPT_LICENSE="true" ;; *) PROMPT_LICENSE="false" ;; esac - if [[ -f ".license_accepted" ]]; then - accepted_license=$(awk -F'|' 'NF >= 5 {print $5; exit}' .license_accepted) - accepted_license="${accepted_license#\"}" ; accepted_license="${accepted_license%\"}" - accepted_license="${accepted_license#\'}" ; accepted_license="${accepted_license%\'}" - fi - - if [[ "${LICENSE_KEY_PROVIDED}" != "true" && -n "${accepted_license}" ]]; then - LICENSE_KEY="${accepted_license}" - export LICENSE_KEY - return 0 - fi - - if [[ "${PROMPT_LICENSE}" == "true" && "${LICENSE_KEY_PROVIDED}" != "true" ]]; then - LICENSE_KEY="" - fi - + # ── Resolve key from file ───────────────────────────────────────── if [[ -z "${LICENSE_KEY}" && -f "${env_file}" ]]; then LICENSE_KEY=$(_get_config_value "$env_file" "LICENSE_KEY") + [[ -n "${LICENSE_KEY}" ]] && license_from_file=true elif [[ -z "${LICENSE_KEY}" && -f "${single_file}" ]]; then LICENSE_KEY=$(_get_config_value "$single_file" "LICENSE_KEY") + [[ -n "${LICENSE_KEY}" ]] && license_from_file=true fi LICENSE_KEY="${LICENSE_KEY#\"}" ; LICENSE_KEY="${LICENSE_KEY%\"}" LICENSE_KEY="${LICENSE_KEY#\'}" ; LICENSE_KEY="${LICENSE_KEY%\'}" - if [[ -z "${LICENSE_KEY}" && "${PROMPT_LICENSE}" != "true" ]] ; then + # ── If key came from file, skip the entire acceptance process ───── + if [[ "${license_from_file}" == "true" && "${LICENSE_KEY_PROVIDED}" != "true" ]]; then + export LICENSE_KEY + return 0 + fi + + # ── No key yet — prompt if possible ────────────────────────────── + if [[ -z "${LICENSE_KEY}" ]]; then if [[ -t 0 || -r /dev/tty ]]; then echo "A license key is required to deploy AnyLog." read -r -p "License Key: " LICENSE_KEY = 5 {print $5; exit}' .license_accepted) - accepted_license="${accepted_license#\"}" ; accepted_license="${accepted_license%\"}" - accepted_license="${accepted_license#\'}" ; accepted_license="${accepted_license%\'}" - fi - - if [[ "${LICENSE_KEY_PROVIDED}" != "true" && -n "${accepted_license}" ]]; then - LICENSE_KEY="${accepted_license}" - export LICENSE_KEY - return 0 - fi - - if [[ "${PROMPT_LICENSE}" == "true" && "${LICENSE_KEY_PROVIDED}" != "true" ]]; then - LICENSE_KEY="" - fi - + # ── Resolve key from file ───────────────────────────────────────── if [[ -z "${LICENSE_KEY}" && -f "${env_file}" ]]; then LICENSE_KEY=$(_get_config_value "$env_file" "LICENSE_KEY") + [[ -n "${LICENSE_KEY}" ]] && license_from_file=true elif [[ -z "${LICENSE_KEY}" && -f "${single_file}" ]]; then LICENSE_KEY=$(_get_config_value "$single_file" "LICENSE_KEY") + [[ -n "${LICENSE_KEY}" ]] && license_from_file=true fi LICENSE_KEY="${LICENSE_KEY#\"}" ; LICENSE_KEY="${LICENSE_KEY%\"}" LICENSE_KEY="${LICENSE_KEY#\'}" ; LICENSE_KEY="${LICENSE_KEY%\'}" - if [[ -z "${LICENSE_KEY}" && "${PROMPT_LICENSE}" != "true" ]] ; then + # ── If key came from file, skip the entire acceptance process ───── + if [[ "${license_from_file}" == "true" && "${LICENSE_KEY_PROVIDED}" != "true" ]]; then + export LICENSE_KEY + return 0 + fi + + # ── No key yet — prompt if possible ────────────────────────────── + if [[ -z "${LICENSE_KEY}" ]]; then if [[ -t 0 || -r /dev/tty ]]; then echo "A license key is required to deploy AnyLog." read -r -p "License Key: " LICENSE_KEY ={GRPC_GENERATED_VERSION}.' - + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' - + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' - ) - - -class PersistenceServiceStub(object): - """Missing associated documentation comment in .proto file.""" - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.WatchTable = channel.unary_stream( - '/persistence.PersistenceService/WatchTable', - request_serializer=dados__pb2.WatchRequest.SerializeToString, - response_deserializer=dados__pb2.DataRow.FromString, - _registered_method=True) - - -class PersistenceServiceServicer(object): - """Missing associated documentation comment in .proto file.""" - - def WatchTable(self, request, context): - """Missing associated documentation comment in .proto file.""" - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - -def add_PersistenceServiceServicer_to_server(servicer, server): - rpc_method_handlers = { - 'WatchTable': grpc.unary_stream_rpc_method_handler( - servicer.WatchTable, - request_deserializer=dados__pb2.WatchRequest.FromString, - response_serializer=dados__pb2.DataRow.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - 'persistence.PersistenceService', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler,)) - server.add_registered_method_handlers('persistence.PersistenceService', rpc_method_handlers) - - - # This class is part of an EXPERIMENTAL API. -class PersistenceService(object): - """Missing associated documentation comment in .proto file.""" - - @staticmethod - def WatchTable(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_stream( - request, - target, - '/persistence.PersistenceService/WatchTable', - dados__pb2.WatchRequest.SerializeToString, - dados__pb2.DataRow.FromString, - options, - channel_credentials, - insecure, - call_credentials, - compression, - wait_for_ready, - timeout, - metadata, - _registered_method=True) diff --git a/customers/grpc_client.al b/customers/grpc_client.al deleted file mode 100644 index 0b26a44b..00000000 --- a/customers/grpc_client.al +++ /dev/null @@ -1,14 +0,0 @@ - \ No newline at end of file diff --git a/docker-makefiles/anylog-generic/node_configs.env b/docker-makefiles/anylog-generic/node_configs.env index 2af268ca..87b68ed1 100644 --- a/docker-makefiles/anylog-generic/node_configs.env +++ b/docker-makefiles/anylog-generic/node_configs.env @@ -282,7 +282,7 @@ BUCKET_SECRET_KEY="" # Whether to enable Remote-GUI in deployment ENABLE_REMOTE_GUI=false # Remote-GUI version Tag -REMOTE_GUI_TAG=beta2 +REMOTE_GUI_TAG=1.1.0 # NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 REMOTE_GUI_NIC="" # Remote GUI Frontend Port diff --git a/docker-makefiles/anylog-master/node_configs.env b/docker-makefiles/anylog-master/node_configs.env index f5dd3bcb..86c7f1e3 100644 --- a/docker-makefiles/anylog-master/node_configs.env +++ b/docker-makefiles/anylog-master/node_configs.env @@ -281,7 +281,7 @@ BUCKET_SECRET_KEY="" # Whether to enable Remote-GUI in deployment ENABLE_REMOTE_GUI=false # Remote-GUI version Tag -REMOTE_GUI_TAG=beta2 +REMOTE_GUI_TAG=1.1.0 # NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 REMOTE_GUI_NIC="" # Remote GUI Frontend Port diff --git a/docker-makefiles/anylog-operator/node_configs.env b/docker-makefiles/anylog-operator/node_configs.env index 1edc8566..730b6054 100644 --- a/docker-makefiles/anylog-operator/node_configs.env +++ b/docker-makefiles/anylog-operator/node_configs.env @@ -14,10 +14,10 @@ IMAGE=anylogco/anylog-network # the repo if needed OR a local path for deployment-scripts and then use that instead of cloning into the repositroy DEPLOYMENTS_REPO=https://github.com/AnyLog-co/deployment-scripts # Branch associated with git repo -DEPLOYMENTS_BRANCH="pre-develop" +DEPLOYMENTS_BRANCH=pre-develop # specify the docker socket - to be used when deploying docker monitoring -DOCKER_SOCKET="/var/run/docker.sock" +DOCKER_SOCKET=/var/run/docker.sock #==== basic ==== @@ -147,9 +147,9 @@ MSG_VALUE_COLUMN_TYPE=float # Whether or not to enable to OPC-UA service ENABLE_OPCUA=false # OPC-UA URL address (ex. opcua.tcp:;//127.0.0.1:4840) -OPCUA_URL="" +OPCUA_URL= # Node information the root is located in (ex. ns=2;s=DataSet) -OPCUA_NODE="" +OPCUA_NODE= # How often to pull data from OPC-UA OPCUA_FREQUENCY=5 @@ -160,8 +160,8 @@ OPCUA_FREQUENCY=5 ANYLOG_PATH=/app # !local_scripts: ${ANYLOG_HOME}/deployment-scripts/scripts LOCAL_SCRIPTS=/app/deployment-scripts -# !test_dir: ${ANYLOG_HOME}/deployment-scripts/test -TEST_DIR=/app/deployment-scripts/test +# !test_dir: writable test output directory +TEST_DIR=/app/AnyLog-Network/data/test #--- General --- # hide blockchain policy - supports all node types except cluster / operator @@ -169,29 +169,29 @@ IS_HIDDEN=false # Whether to automatically run a local (or personalized) script at the end of the process DEPLOY_LOCAL_SCRIPT=false # Coordinates of the machine - used by Grafana to map the network -LOCATION="" +LOCATION= # Country where machine is located -COUNTRY="" +COUNTRY= # State where machine is located -STATE="" +STATE= # City where machine is located -CITY="" +CITY= #--- Networking --- # Declare Policy name -CONFIG_NAME="" +CONFIG_NAME= # Whether to enable DNS resolution ENABLE_DNS=false # NIC type used to resolve the internal IP address (ex. eth0, lo) -NIC_TYPE="" +NIC_TYPE= # Overlay IP address - if set, will replace local IP address when connecting to network -OVERLAY_IP="" +OVERLAY_IP= # External DNS resolution -EXTERNAL_DNS="" +EXTERNAL_DNS= # local DNS domain to append to hostname (ex. example.com) -DNS_DOMAIN="" +DNS_DOMAIN= # Full path for machine's (local) DNS -DNS="" +DNS= # Bind settings - a true value binds to a specific IP and Port (a false value binds to all IPs) TCP_BIND=true @@ -214,20 +214,20 @@ BLOBS_COMPRESS=true # Folder for storing blobs locally # Leave empty to use default application path -BLOBS_FOLDER="" +BLOBS_FOLDER= #--- Object Store --- # Logical bucket grouping -BUCKET_GROUP="" +BUCKET_GROUP= # Bucket identifier -BUCKET_ID="" +BUCKET_ID= # Cloud region -BUCKET_REGION="" +BUCKET_REGION= #--- Operator --- # Operator ID - autogenerated INT if not specified # This is used to help keep a record of where data was originally stored -MEMBER="" +MEMBER= # Whether of not to enable HA against the cluster ENABLE_HA=false # How many days back to sync between nodes @@ -253,7 +253,7 @@ QUERY_POOL=6 #--- General --- # AnyLog License Key -LICENSE_KEY="" +LICENSE_KEY= #--- Database --- # Username for SQL database connection @@ -274,22 +274,22 @@ MQTT_PASSWD=mqtt4AnyLog! #--- Object Store --- # Access credentials -BUCKET_ACCESS_KEY="" -BUCKET_SECRET_KEY="" +BUCKET_ACCESS_KEY= +BUCKET_SECRET_KEY= #==== remote-gui ==== # Companion UI application - deployed alongside the AnyLog node # Whether to enable Remote-GUI in deployment ENABLE_REMOTE_GUI=false # Remote-GUI version Tag -REMOTE_GUI_TAG=beta2 +REMOTE_GUI_TAG=1.1.0 # NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 -REMOTE_GUI_NIC="" +REMOTE_GUI_NIC= # Remote GUI Frontend Port REMOTE_GUI_FE=31800 # Remote GUI Backend Port REMOTE_GUI_BE=8080 # Grafana PATH to reference for Grafana connection -GRAFANA_URL="" +GRAFANA_URL= diff --git a/docker-makefiles/anylog-publisher/node_configs.env b/docker-makefiles/anylog-publisher/node_configs.env index e49a862b..69971810 100644 --- a/docker-makefiles/anylog-publisher/node_configs.env +++ b/docker-makefiles/anylog-publisher/node_configs.env @@ -282,7 +282,7 @@ BUCKET_SECRET_KEY="" # Whether to enable Remote-GUI in deployment ENABLE_REMOTE_GUI=false # Remote-GUI version Tag -REMOTE_GUI_TAG=beta2 +REMOTE_GUI_TAG=1.1.0 # NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 REMOTE_GUI_NIC="" # Remote GUI Frontend Port diff --git a/docker-makefiles/anylog-query/node_configs.env b/docker-makefiles/anylog-query/node_configs.env index fc7bc875..305e80c1 100644 --- a/docker-makefiles/anylog-query/node_configs.env +++ b/docker-makefiles/anylog-query/node_configs.env @@ -282,7 +282,7 @@ BUCKET_SECRET_KEY="" # Whether to enable Remote-GUI in deployment ENABLE_REMOTE_GUI=true # Remote-GUI version Tag -REMOTE_GUI_TAG=beta2 +REMOTE_GUI_TAG=1.1.0 # NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 REMOTE_GUI_NIC="" # Remote GUI Frontend Port diff --git a/docker-makefiles/anylog-standalone-operator/node_configs.env b/docker-makefiles/anylog-standalone-operator/node_configs.env index ebb760b7..05cc8fd4 100644 --- a/docker-makefiles/anylog-standalone-operator/node_configs.env +++ b/docker-makefiles/anylog-standalone-operator/node_configs.env @@ -12,38 +12,18 @@ IMAGE=anylogco/anylog-network # Github repo to use for configuring / deploying node # The path can either be a Github URL and then the code will automatically clone # the repo if needed OR a local path for deployment-scripts and then use that instead of cloning into the repositroy +#DEPLOYMENTS_REPO=https://github.com/AnyLog-co/deployment-scripts +DEPLOYMENTS_REPO=/Users/mdavidson58/Documents/Anylog/pre-develop/deployment-scripts # Branch associated with git repo -DEPLOYMENTS_REPO="/mnt/c/Users/oshad/AnyLog-code/deployment-scripts" -DEPLOYMENTS_BRANCH="os-dev" +DEPLOYMENTS_BRANCH=os-dev # specify the docker socket - to be used when deploying docker monitoring -DOCKER_SOCKET="/var/run/docker.sock" - -# Local folders to mount inside the AnyLog container. -# -# Examples: -# No volumes: -# USER_VOLUMES="" -# -# Single volume: -# USER_VOLUMES="/home/user/grpc-scripts" -# -# Multiple volumes (space-separated): -# USER_VOLUMES="/home/user/grpc-scripts /home/user/local-deployments" -# -# Notes: -# - Use absolute paths only -# - Paths are mounted automatically under /app/ -# -# Example: -# /home/user/grpc-scripts -> /app/grpc-scripts -# -USER_VOLUMES="/mnt/c/Users/oshad/AnyLog-code/docker-compose/customers" +DOCKER_SOCKET=/var/run/docker.sock #==== basic ==== # Core node configuration - required for all deployments #--- General --- -# Configuration based deployment node deployment type +# Configuration based deploymnet node deployment type # Node Types: # generic: sandbox with only network configured # master: "blockchain" emulator @@ -109,24 +89,20 @@ BLOCKCHAIN_DESTINATION=file # Owner of the cluster CLUSTER_NAME=anylog-cluster1 # Logical database name -DEFAULT_DBMS=persistence +DEFAULT_DBMS=new_company #==== southbound ==== # Data ingestion protocols (MQTT, OPC-UA, video streaming, monitoring) - #--- Monitoring --- # Whether to monitor the node or not -NODE_MONITORING=true +NODE_MONITORING=false # Store monitoring in Operator node(s) -STORE_MONITORING=true +STORE_MONITORING=false # Accept syslog data from local (Message broker required) SYSLOG_MONITORING=true # Docker Monitoring DOCKER_MONITORING=true -# how often data is being pulled from docker -DOCKER_FREQUENCY = 30 - #--- Video Streaming --- # Enable or disable video streaming. # true = stream video, false = do not stream @@ -145,7 +121,7 @@ YOLO_MODEL_PORT=50051 #--- MQTT --- # Whether to enable the default MQTT process -ENABLE_MQTT=false +ENABLE_MQTT=true # IP address of MQTT broker MQTT_BROKER=172.104.228.251 @@ -173,7 +149,7 @@ ENABLE_OPCUA=true # OPC-UA URL address (ex. opcua.tcp:;//127.0.0.1:4840) OPCUA_URL=opc.tcp://virtualfactory.proveit.services:4842/discovery # Node information the root is located in (ex. ns=2;s=DataSet) -OPCUA_NODE="ns=2;s=Site1" +OPCUA_NODE=ns=2;s=Site1 # How often to pull data from OPC-UA OPCUA_FREQUENCY=5 @@ -184,41 +160,41 @@ OPCUA_FREQUENCY=5 ANYLOG_PATH=/app # !local_scripts: ${ANYLOG_HOME}/deployment-scripts/scripts LOCAL_SCRIPTS=/app/deployment-scripts -# !test_dir: ${ANYLOG_HOME}/deployment-scripts/test -TEST_DIR=/app/deployment-scripts/test +# !test_dir: writable test output directory +TEST_DIR=/app/AnyLog-Network/data/test #--- General --- # hide blockchain policy - supports all node types except cluster / operator IS_HIDDEN=false # Whether to automatically run a local (or personalized) script at the end of the process -DEPLOY_LOCAL_SCRIPT=false +DEPLOY_LOCAL_SCRIPT=true # Coordinates of the machine - used by Grafana to map the network -LOCATION="" +LOCATION= # Country where machine is located -COUNTRY="" +COUNTRY= # State where machine is located -STATE="" +STATE= # City where machine is located -CITY="" +CITY= #--- Networking --- # Declare Policy name -CONFIG_NAME="" +CONFIG_NAME= # Whether to enable DNS resolution ENABLE_DNS=false # NIC type used to resolve the internal IP address (ex. eth0, lo) -NIC_TYPE="" +NIC_TYPE= # Overlay IP address - if set, will replace local IP address when connecting to network -OVERLAY_IP="" +OVERLAY_IP= # External DNS resolution -EXTERNAL_DNS="" +EXTERNAL_DNS= # local DNS domain to append to hostname (ex. example.com) -DNS_DOMAIN="" +DNS_DOMAIN= # Full path for machine's (local) DNS -DNS="" +DNS= # Bind settings - a true value binds to a specific IP and Port (a false value binds to all IPs) -TCP_BIND=false +TCP_BIND=true REST_BIND=false BROKER_BIND=false @@ -227,8 +203,7 @@ TCP_THREADS=6 REST_THREADS=6 BROKER_THREADS=6 -# Timeout in seconds for REST requests (0 = continuous) -REST_TIMEOUT=30 +# ST_TIMEOUT=30 #--- NoSQL --- # Reuse identical blobs using file hash @@ -238,40 +213,38 @@ BLOBS_COMPRESS=true # Folder for storing blobs locally # Leave empty to use default application path -BLOBS_FOLDER="" +BLOBS_FOLDER= #--- Object Store --- # Logical bucket grouping -BUCKET_GROUP="" +BUCKET_GROUP= # Bucket identifier -BUCKET_ID="" +BUCKET_ID= # Cloud region -BUCKET_REGION="" +BUCKET_REGION= #--- Operator --- # Operator ID - autogenerated INT if not specified # This is used to help keep a record of where data was originally stored -MEMBER="" +MEMBER= # Whether of not to enable HA against the cluster ENABLE_HA=false # How many days back to sync between nodes START_DATE=30 # How many threads to use in the operator process OPERATOR_THREADS=3 -# Parallel threads to be used in order digest data into AnyLog -OPERATOR_HELPERS=5 + #--- Sub processes --- # Compress JSON and SQL file(s) backup COMPRESS_FILE=true # When data comes in write to database immediately, as opposed to waiting for a full buffer WRITE_IMMEDIATE=true # If buffer is not full, how long to wait until pushing data through -THRESHOLD_TIME=10 seconds +THRESHOLD_TIME=60 seconds # Buffer size to reach, at which point data is pushed through -THRESHOLD_VOLUME=1KB +THRESHOLD_VOLUME=10KB # Number of parallel queries QUERY_POOL=6 -# for `run operator` process, specify the number of parallel threads to use #==== secrets ==== # Sensitive credentials - do not commit to version control @@ -300,23 +273,23 @@ MQTT_PASSWD=mqtt4AnyLog! #--- Object Store --- # Access credentials -BUCKET_ACCESS_KEY="" -BUCKET_SECRET_KEY="" +BUCKET_ACCESS_KEY= +BUCKET_SECRET_KEY= #==== remote-gui ==== # Companion UI application - deployed alongside the AnyLog node # Whether to enable Remote-GUI in deployment -ENABLE_REMOTE_GUI=false +ENABLE_REMOTE_GUI=true # Remote-GUI version Tag -REMOTE_GUI_TAG=beta2 +REMOTE_GUI_TAG=1.1.0 # NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 -REMOTE_GUI_NIC="" +REMOTE_GUI_NIC= # Remote GUI Frontend Port REMOTE_GUI_FE=31800 # Remote GUI Backend Port REMOTE_GUI_BE=8080 # Default connection to REST service (IP:Port) if not provided then autogenerated using `REMOTE_GUI_NIC` -REMOTE_CONN="" +REMOTE_CONN= # Grafana PATH to reference for Grafana connection -GRAFANA_URL="" +GRAFANA_URL= diff --git a/docker-makefiles/anylog-standalone-publisher/node_configs.env b/docker-makefiles/anylog-standalone-publisher/node_configs.env index 78003efb..20c673e8 100644 --- a/docker-makefiles/anylog-standalone-publisher/node_configs.env +++ b/docker-makefiles/anylog-standalone-publisher/node_configs.env @@ -281,7 +281,7 @@ BUCKET_SECRET_KEY="" # Whether to enable Remote-GUI in deployment ENABLE_REMOTE_GUI=true # Remote-GUI version Tag -REMOTE_GUI_TAG=beta2 +REMOTE_GUI_TAG=1.1.0 # NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 REMOTE_GUI_NIC="" # Remote GUI Frontend Port diff --git a/docker-makefiles/prep_configs.sh b/docker-makefiles/prep_configs.sh index c3c4f30a..f1c230ac 100644 --- a/docker-makefiles/prep_configs.sh +++ b/docker-makefiles/prep_configs.sh @@ -47,8 +47,9 @@ fi normalize_quotes() { local file="$1" echo "Normalizing quotes in: ${file}" - sedi "s/=''/=\"\"/g" "${file}" - sedi -E "s/^([A-Za-z_][A-Za-z0-9_]*)='(.*)'/\1=\"\2\"/" "${file}" + sedi "s/=''$/=/g" "${file}" + sedi -E "s/^([A-Za-z_][A-Za-z0-9_]*)='(.*)'/\1=\2/" "${file}" + sedi -E "s/^([A-Za-z_][A-Za-z0-9_]*)=\"(.*)\"/\1=\2/" "${file}" } for cfg in "${CONFIG_FILES[@]}"; do diff --git a/openbao/docker-compose.yml b/openbao/docker-compose.yml deleted file mode 100644 index 5b195380..00000000 --- a/openbao/docker-compose.yml +++ /dev/null @@ -1,24 +0,0 @@ -services: - anylog-node: - image: anylogco/anylog-network:1.3.2412-beta8 - restart: always - container_name: anylog-node - stdin_open: true - tty: true - network_mode: host - environment: - - INIT_TYPE=prod - - OPENBAO_VAULT_IP=http://0.0.0.0:8200 - - OPENBAO_VAULT_TOKEN=s.57hG4fLMZzRbbPF7mF5fA2Md - - SECTION_NAME=mybao - volumes: - - anylog-node-anylog:/app/AnyLog-Network/anylog - - anylog-node-blockchain:/app/AnyLog-Network/blockchain - - anylog-node-data:/app/AnyLog-Network/data - - anylog-node-local-scripts:/app/deployment-scripts - -volumes: - anylog-node-anylog: - anylog-node-blockchain: - anylog-node-data: - anylog-node-local-scripts: \ No newline at end of file diff --git a/openbao/get4bao.sh b/openbao/get4bao.sh deleted file mode 100644 index 9a2629b8..00000000 --- a/openbao/get4bao.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -# 0.0.0.0:8200 s.Pb8yAsBHAKyva0DHmQA4Ib12 -python3 get4bao.py ${VAULT_URL} ${VAULT_TOKEN} - -source $ANYLOG_PATH/openbao/configs.env \ No newline at end of file diff --git a/openbao/openbao.env b/openbao/openbao.env deleted file mode 100644 index 9bd4ed5f..00000000 --- a/openbao/openbao.env +++ /dev/null @@ -1,9 +0,0 @@ -# test configs -SERVICE_PORT=32548 -TCP_BIND=true -TCP_THREADS=3 - -REST_PORT=32549 -REST_TIMEOUT=30 -REST_BIND=false -REST_THREADS=3 \ No newline at end of file diff --git a/openbao/openbao_cmds.py b/openbao/openbao_cmds.py deleted file mode 100644 index 3b246854..00000000 --- a/openbao/openbao_cmds.py +++ /dev/null @@ -1,270 +0,0 @@ -# URL: https://openbao.org/api-docs/ -import argparse -import ast -import json -import os -import requests - -ERRORS_GENERIC = { - 1: "Informational", - 2: "Successful", - 3: "Redirection", - 4: "Client Error", - 5: "Server Error", - 7: "Developer Error" -} - -NETWORK_ERRORS = { - 100: "Continue", - 101: "Switching Protocols", - 200: "OK", - 201: "Created", - 202: "Accepted", - 203: "Non-Authoritative Information", - 204: "No Content", - 205: "Reset Content", - 206: "Partial Content", - 300: "Multiple Choices", - 301: "Moved Permanently", - 302: "Found", - 303: "See Other", - 304: "Not Modified", - 305: "Use Proxy", - 307: "Temporary Redirect", - 400: "Bad Request", - 401: "Unauthorized", - 402: "Payment Required", - 403: "Forbidden", - 404: "Not Found", - 405: "Method Not Allowed", - 406: "Not Acceptable", - 407: "Proxy Authentication Required", - 408: "Request Timeout", - 409: "Conflict", - 410: "Gone", - 411: "Length Required", - 412: "Precondition Failed", - 413: "Payload Too Large", - 414: "URI Too Long", - 415: "Unsupported Media Type", - 416: "Range Not Satisfiable", - 417: "Expectation Failed", - 418: "I'm a teapot", - 426: "Upgrade Required", - 500: "Internal Server Error", - 501: "Not Implemented", - 502: "Bad Gateway", - 503: "Service Unavailable", - 504: "Gateway Time-out", - 505: "HTTP Version Not Supported", - 102: "Processing", - 207: "Multi-Status", - 226: "IM Used", - 308: "Permanent Redirect", - 422: "Unprocessable Entity", - 423: "Locked", - 424: "Failed Dependency", - 428: "Precondition Required", - 429: "Too Many Requests", - 431: "Request Header Fields Too Large", - 451: "Unavailable For Legal Reasons", - 506: "Variant Also Negotiates", - 507: "Insufficient Storage", - 511: "Network Authentication Required" -} - -DEFAULT_CONFIG_FILE = os.path.join(os.path.expandvars(os.path.expanduser(__file__.split("openbao_cmds.py")[0])), 'openbao.env') - - -class OpenBao: - def __init__(self, conn:str, token:str): - self.conn = conn if conn is not None else '0.0.0.0:8200' - self.token = token - if self.token is None: - raise ValueError("Missing token value, cannot continue...") - - def __execute_get(self, url:str, headers:dict): - """ - Execute GET request - :args: - url:str - request URl - headers:dict - REST headers - :params: - response:requests.GET - response fromm GET - :return: - response - if response.status_code has a Client Error, return empty dict - """ - try: - response = requests.get(url=url, headers=headers) - except Exception as error: - raise Exception(f"Failed to execute GET against {url} (Error: {error}") - else: - if 400 <= response.status_code < 500: - response = {} - elif not 200 <= response.status_code < 300: - if int(response.status_code) in NETWORK_ERRORS: - raise ConnectionError(f"Failed to execute GET against {url} (Network Error {response.status_code}: {NETWORK_ERRORS[int(response.status_code)]})") - elif int(response.status_code[0]) in ERRORS_GENERIC: - raise ConnectionError(f"Failed to execute GET against {url} (Network Error {response.status_code}: {ERRORS_GENERIC[int(response.status_code[0])]}") - - return response - - - def get_openbao_status(self): - """ - health check for OpenBao - :anylog-cmd: - get openbao status where name=mybao - :args: - openbao_name:str - OpenBao connection name - :params: - headers:dict - REST headers - url:str - URL to access OpenBao - :return: - dictionary for OpenBao status - :sample output: - { - 'initialized': True, 'sealed': False, 'standby': False, 'performance_standby': False, - 'replication_performance_mode': 'disabled', 'replication_dr_mode': 'disabled', - 'server_time_utc': 1738794866, 'version': '2.1.1', 'cluster_name': 'vault-cluster-8acc91c5', - 'cluster_id': 'bd9670ca-b594-3c6b-6431-2d81afa9bdd0' - } - """ - headers = { - "X-Vault-Token": self.token, - "Content-Type": "application/json" - } - - url = f"http://{self.conn}/v1/sys/health" - response = self.__execute_get(url=url, headers=headers) - try: - return response.json() - except Exception as error: - raise Exception(f"Failed to extract results for {self.conn} (Error: {error})") - - - def get_openbao_value(self, section:str, key:str=None): - """ - Get secret value(s) within OpenBao - :anylog-cmd: - get openbao status where name=mybao and section=XXX [and key=XXX] - :args: - openbao_name:str - OpenBao connection name - section:str - subsection within secrets data key/value pairs will be stored - key:str - specific key to get value for - :params: - headers:dict - REST headers - url:str - URL to access OpenBao - :return: - if key - return specific value - if no key - return full dict of params - :note: - if section or key is invalid then returns error 404 - error should be "failed to get valid data" - """ - headers = { - "X-Vault-Token": self.token, - } - - url = f"http://{self.conn}/v1/secret/data/{section}" - response = self.__execute_get(url=url, headers=headers) - if response == {}: - return response - - try: - output = response.json() - except Exception as error: - raise Exception(f"Failed to extract key/values for {url} (Error: {error})") - - content = output['data']['data'][section] - if key: - if key in content: - return content[key] - else: - raise ValueError(f'Failed to find value for {key}') - else: - return content - - - def publish_data(self, section:str, payload:dict): - """ - Publish value to OpenBao - :args: - :args: - openbao_name:str - OpenBao connection name - section:str - subsection within secrets data key/value pairs will be stored - payload:dict - key/value pairs - :params: - headers:dict - REST headers - url:str - URL to access OpenBao - """ - headers = { - "X-Vault-Token": self.token, - "Content-Type": "application/json" - } - - orig_payloads = self.get_openbao_value(section=section) - updated_payloads = {**orig_payloads, **payload} - publish_payloads = {'data': { - section: updated_payloads - }} - - url = f"http://{self.conn}/v1/secret/data/{section}" - - try: - response = requests.post(url=url, headers=headers, json=publish_payloads) - except Exception as error: - raise Exception(f"Failed to POST data against {url} (Error: {error})") - else: - if not 200 <= response.status_code < 300: - if int(response.status_code) in NETWORK_ERRORS: - raise ConnectionError(f"Failed to execute POST against {url} (Network Error {response.status_code}: {NETWORK_ERRORS[int(response.status_code)]})") - elif int(response.status_code[0]) in ERRORS_GENERIC: - raise ConnectionError(f"Failed to execute POST against {url} (Network Error {response.status_code}: {ERRORS_GENERIC[int(response.status_code[0])]}") - - -def main(): - parse = argparse.ArgumentParser() - parse.add_argument('--config-file', type=str,default=DEFAULT_CONFIG_FILE, help='OpenBao .env config files') - parse.add_argument('--conn', type=str, default='0.0.0.0:8200', help='URL connection') - parse.add_argument('--token', type=str, default=None, help='OpenBao token') - parse.add_argument('--section', type=str, default='mybao', help='Section to secrets data location') - args = parse.parse_args() - - openbao = OpenBao(conn=args.conn, token=args.token) - args.config_file = os.path.expandvars(os.path.expanduser(args.config_file)) - if not os.path.isfile(args.config_file): - raise FileNotFoundError(f"Failed to locate config file {args.config_file}") - - # check status - print(openbao.get_openbao_status()) - configs = {} - with open(args.config_file, 'r') as f: - for line in f.read().split("\n"): - if not (line.startswith("#") or not line): - key, value = line.split("=") - try: - configs[key.strip()] = ast.literal_eval(value.strip()) - except: - configs[key.strip()] = value.strip() - - openbao.publish_data(section=args.section, payload=configs) - print(openbao.get_openbao_value(section=args.section)) - - -if __name__ == '__main__': - main() - # # 127.0.0.1:8200 --check-health - # openbao = OpenBao() - # openbao.connect_openbao(name='mybao', conn='0.0.0.0:8200', token='s.57hG4fLMZzRbbPF7mF5fA2Md') - # - # # check status - # print(openbao.get_openbao_status(openbao_name='mybao')) - # - # # create section and set payload value(s) - # openbao.publish_data(openba_name='mybao', section='anylog-data', payload={'param1': True, 'param2': 3, 'param3': 'abc'}) - # print(openbao.get_openbao_value(openbao_name='mybao', section='anylog-data')) - # - # # add another value and update value to payload - # openbao.publish_data(openba_name='mybao', section='anylog-data', payload={'param1': False, 'param4': ''}) - # print(openbao.get_openbao_value(openbao_name='mybao', section='anylog-data')) \ No newline at end of file diff --git a/openbao/run_openbao.sh b/openbao/run_openbao.sh deleted file mode 100644 index 9caa0d63..00000000 --- a/openbao/run_openbao.sh +++ /dev/null @@ -1,7 +0,0 @@ -docker run -d \ - --name openbao \ - -p 8080:8080 \ - -p 8200:8200 \ - -v openbao-config:/config \ - -v openbao-data:/data \ ---rm openbao/openbao:latest \ No newline at end of file From 4516087537cac288b9e6d68f7ca8622b98a9b549 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 13 May 2026 23:06:34 +0000 Subject: [PATCH 35/46] chore: update version for os-dev [skip ci] --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 182e81e9..2b83fe55 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = docker-compose -version = 1.4.2604 (os-dev - d86149 [2026-05-13 21:41:39]) +version = 1.4.2604 (os-dev - 71ebae [2026-05-13 23:06:34]) author = AnyLog Co. contact = info@anylog.co description = docker-compose configurations for AnyLog deployment From 94d35752cb9d47190e5c4dd2f01f407fec7308cf Mon Sep 17 00:00:00 2001 From: Ori Shadmon Date: Wed, 13 May 2026 18:00:45 -0700 Subject: [PATCH 36/46] Baraba for Mark Signed-off-by: Ori Shadmon --- customers/barbara.json | 360 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 360 insertions(+) create mode 100644 customers/barbara.json diff --git a/customers/barbara.json b/customers/barbara.json new file mode 100644 index 00000000..31d835e1 --- /dev/null +++ b/customers/barbara.json @@ -0,0 +1,360 @@ +{ + "appName": "anylog-standalone-operator", + "version": "1.4.2604", + "name": "AnyLog Standalone Operator", + "architectures": "x86_64,aarch64", + "type": "marketplace", + "releaseNotes": [ + "Initial AnyLog standalone operator marketplace template" + ], + "templateAppconfig": {}, + "uris": [ + { + "name": "AnyLog REST", + "serviceName": "anylog-standalone-operator", + "pattern": "http://:", + "link": true + } + ], + "services": [ + { + "name": "anylog-standalone-operator", + "ports": [ + { + "description": "TCP port for AnyLog node-to-node protocol", + "name": "ANYLOG_SERVER_PORT", + "readableName": "AnyLog server (TCP) port", + "type": "text", + "default": "32148", + "mandatory": true, + "regex": "^(?:[1-9]\\d{0,3}|[1-5]\\d{4}|6[0-4]\\d{3}|65[0-4]\\d{2}|655[0-2]\\d|6553[0-5])$" + }, + { + "description": "HTTP REST API port for AnyLog", + "name": "ANYLOG_REST_PORT", + "readableName": "AnyLog REST port", + "type": "text", + "default": "32149", + "mandatory": true, + "regex": "^(?:[1-9]\\d{0,3}|[1-5]\\d{4}|6[0-4]\\d{3}|65[0-4]\\d{2}|655[0-2]\\d|6553[0-5])$" + }, + { + "description": "MQTT / broker port used by AnyLog", + "name": "ANYLOG_BROKER_PORT", + "readableName": "AnyLog broker port", + "type": "text", + "default": "32150", + "mandatory": true, + "regex": "^(?:[1-9]\\d{0,3}|[1-5]\\d{4}|6[0-4]\\d{3}|65[0-4]\\d{2}|655[0-2]\\d|6553[0-5])$" + } + ], + "volumes": [], + "env": [ + { + "description": "AnyLog license key (hex payload with trailing JSON metadata)", + "name": "LICENSE_KEY", + "readableName": "License key", + "type": "text", + "default": "", + "mandatory": true, + "regex": "^[0-9a-fA-F]+\\{.*\\}$" + }, + { + "description": "Unique name for this AnyLog instance", + "name": "NODE_NAME", + "readableName": "Node name", + "type": "text", + "default": "anylog-standalone-operator", + "mandatory": true, + "regex": "^[a-zA-Z0-9]([a-zA-Z0-9._-]*[a-zA-Z0-9])?$" + }, + { + "description": "Master ledger endpoint as host:port (IPv4 or hostname)", + "name": "LEDGER_CONN", + "readableName": "Ledger connection", + "type": "text", + "default": "127.0.0.1:32148", + "mandatory": true, + "regex": "^(?:(?:25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)(?:\\.(?:25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)){3}|[a-zA-Z0-9](?:[a-zA-Z0-9._-]*[a-zA-Z0-9])?):(?:[1-9]\\d{0,3}|[1-5]\\d{4}|6[0-4]\\d{3}|65[0-4]\\d{2}|655[0-2]\\d|6553[0-5])$" + }, + { + "description": "Owner or organization label for this AnyLog instance", + "name": "COMPANY_NAME", + "readableName": "Company name", + "type": "text", + "default": "AnyLog Co.", + "mandatory": true, + "regex": "^[^\\r\\n]{1,256}$" + }, + { + "description": "Physical SQL database backend (sqlite or psql)", + "name": "DB_TYPE", + "readableName": "Database type", + "type": "text", + "default": "sqlite", + "mandatory": true, + "regex": "^(sqlite|psql)$" + }, + { + "description": "Host or IP address of the SQL database", + "name": "DB_IP", + "readableName": "Database host", + "type": "text", + "default": "127.0.0.1", + "mandatory": true, + "regex": "^(?:(?:25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)(?:\\.(?:25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)){3}|[a-zA-Z0-9](?:[a-zA-Z0-9._-]*[a-zA-Z0-9])?)$" + }, + { + "description": "TCP port of the SQL database", + "name": "DB_PORT", + "readableName": "Database port", + "type": "text", + "default": "5432", + "mandatory": true, + "regex": "^(?:[1-9]\\d{0,3}|[1-5]\\d{4}|6[0-4]\\d{3}|65[0-4]\\d{2}|655[0-2]\\d|6553[0-5])$" + }, + { + "description": "Default logical DBMS name used by the operator", + "name": "DEFAULT_DBMS", + "readableName": "Default DBMS", + "type": "text", + "default": "new_company", + "mandatory": true, + "regex": "^[a-zA-Z0-9_]{1,64}$" + }, + { + "description": "Whether to enable the default MQTT ingestion process", + "name": "ENABLE_MQTT", + "readableName": "Enable MQTT", + "type": "text", + "default": "true", + "mandatory": true, + "regex": "^(true|false)$" + }, + { + "description": "IP address or hostname of the MQTT broker", + "name": "MQTT_BROKER", + "readableName": "MQTT broker host", + "type": "text", + "default": "172.104.228.251", + "mandatory": false, + "regex": "^(?:(?:25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)(?:\\.(?:25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)){3}|[a-zA-Z0-9](?:[a-zA-Z0-9._-]*[a-zA-Z0-9])?)$" + }, + { + "description": "TCP port of the MQTT broker", + "name": "MQTT_PORT", + "readableName": "MQTT broker port", + "type": "text", + "default": "1883", + "mandatory": false, + "regex": "^(?:[1-9]\\d{0,3}|[1-5]\\d{4}|6[0-4]\\d{3}|65[0-4]\\d{2}|655[0-2]\\d|6553[0-5])$" + }, + { + "description": "Whether to enable MQTT client logging", + "name": "MQTT_LOG", + "readableName": "MQTT logging", + "type": "text", + "default": "false", + "mandatory": true, + "regex": "^(true|false)$" + }, + { + "description": "Username for MQTT broker authentication", + "name": "MQTT_USER", + "readableName": "MQTT user", + "type": "text", + "default": "anyloguser", + "mandatory": false, + "regex": "^[^\\r\\n]{0,128}$" + }, + { + "description": "Password for MQTT broker authentication", + "name": "MQTT_PASSWD", + "readableName": "MQTT password", + "type": "text", + "default": "mqtt4AnyLog!", + "mandatory": false, + "regex": "" + }, + { + "description": "MQTT topic to subscribe for ingested messages", + "name": "MSG_TOPIC", + "readableName": "Message topic", + "type": "text", + "default": "rand-data", + "mandatory": false, + "regex": "^[^\\r\\n]{1,256}$" + }, + { + "description": "Logical database (DBMS) name for MQTT message storage", + "name": "MSG_DBMS", + "readableName": "Message DBMS", + "type": "text", + "default": "mydb", + "mandatory": false, + "regex": "^[a-zA-Z0-9_]{1,64}$" + }, + { + "description": "Target table mapping for MQTT payloads (e.g. bring [table])", + "name": "MSG_TABLE", + "readableName": "Message table", + "type": "text", + "default": "bring [table]", + "mandatory": false, + "regex": "^[^\\r\\n]{1,256}$" + }, + { + "description": "Timestamp column mapping for MQTT payloads", + "name": "MSG_TIMESTAMP_COLUMN", + "readableName": "Message timestamp column", + "type": "text", + "default": "bring [timestamp]", + "mandatory": false, + "regex": "^[^\\r\\n]{1,256}$" + }, + { + "description": "Value column mapping for MQTT payloads", + "name": "MSG_VALUE_COLUMN", + "readableName": "Message value column", + "type": "text", + "default": "bring [value]", + "mandatory": false, + "regex": "^[^\\r\\n]{1,256}$" + }, + { + "description": "SQL type for the value column (e.g. float, int)", + "name": "MSG_VALUE_COLUMN_TYPE", + "readableName": "Message value column type", + "type": "text", + "default": "float", + "mandatory": false, + "regex": "^[a-zA-Z0-9_]{1,32}$" + }, + { + "description": "Whether to enable the OPC-UA client service", + "name": "ENABLE_OPCUA", + "readableName": "Enable OPC-UA", + "type": "text", + "default": "true", + "mandatory": true, + "regex": "^(true|false)$" + }, + { + "description": "OPC-UA server endpoint URL (e.g. opc.tcp://host:4840/...)", + "name": "OPCUA_URL", + "readableName": "OPC-UA URL", + "type": "text", + "default": "opc.tcp://virtualfactory.proveit.services:4842/discovery", + "mandatory": false, + "regex": "^opc\\.tcp://[^\\s\\r\\n]{8,512}$" + }, + { + "description": "OPC-UA node id to read (e.g. ns=2;s=Site1)", + "name": "OPCUA_NODE", + "readableName": "OPC-UA node", + "type": "text", + "default": "ns=2;s=Site1", + "mandatory": false, + "regex": "^[^\\r\\n]{1,512}$" + }, + { + "description": "Polling interval in seconds for OPC-UA reads", + "name": "OPCUA_FREQUENCY", + "readableName": "OPC-UA frequency (seconds)", + "type": "text", + "default": "5", + "mandatory": false, + "regex": "^[1-9][0-9]{0,5}$" + }, + { + "description": "Geographic coordinates for this node (e.g. for Grafana maps)", + "name": "LOCATION", + "readableName": "Location", + "type": "text", + "default": "", + "mandatory": false, + "regex": "^[^\\r\\n]{0,256}$" + }, + { + "description": "Country where this machine is located", + "name": "COUNTRY", + "readableName": "Country", + "type": "text", + "default": "", + "mandatory": false, + "regex": "^[^\\r\\n]{0,128}$" + }, + { + "description": "State or region where this machine is located", + "name": "STATE", + "readableName": "State", + "type": "text", + "default": "", + "mandatory": false, + "regex": "^[^\\r\\n]{0,128}$" + }, + { + "description": "City where this machine is located", + "name": "CITY", + "readableName": "City", + "type": "text", + "default": "", + "mandatory": false, + "regex": "^[^\\r\\n]{0,128}$" + }, + { + "description": "Whether to enable DNS resolution for advertised endpoints", + "name": "ENABLE_DNS", + "readableName": "Enable DNS", + "type": "text", + "default": "false", + "mandatory": true, + "regex": "^(true|false)$" + }, + { + "description": "External DNS server or resolver hint", + "name": "EXTERNAL_DNS", + "readableName": "External DNS", + "type": "text", + "default": "", + "mandatory": false, + "regex": "^[^\\r\\n]{0,256}$" + }, + { + "description": "DNS domain appended to hostnames (e.g. example.com)", + "name": "DNS_DOMAIN", + "readableName": "DNS domain", + "type": "text", + "default": "", + "mandatory": false, + "regex": "^([a-zA-Z0-9._-]{0,253})?$" + }, + { + "description": "Path or URI for local DNS configuration used by the node", + "name": "DNS", + "readableName": "DNS configuration", + "type": "text", + "default": "", + "mandatory": false, + "regex": "^[^\\r\\n]{0,512}$" + }, + { + "description": "Network interface used to resolve the node IP (e.g. eth0, lo); leave empty for default", + "name": "NIC_TYPE", + "readableName": "NIC type", + "type": "text", + "default": "", + "mandatory": false, + "regex": "^([a-zA-Z0-9._-]{1,15})?$" + } + ] + } + ], + "marketDescription": { + "categories": [], + "developer": "AnyLog", + "docker": true, + "longDesc": "AnyLog network node (standalone operator): TCP, REST, and broker ports with core environment configuration.", + "shortDesc": "AnyLog standalone operator node." + } +} From 776744e66c1ef07f9a32ae3373c12bf81d4f21b8 Mon Sep 17 00:00:00 2001 From: Ori Shadmon Date: Fri, 15 May 2026 16:08:26 -0700 Subject: [PATCH 37/46] cleanup to match mark's config files Signed-off-by: Ori Shadmon --- docker-makefiles/prep_configs.sh | 38 +++++++++++++------------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/docker-makefiles/prep_configs.sh b/docker-makefiles/prep_configs.sh index 5d9a0d2c..30d88654 100644 --- a/docker-makefiles/prep_configs.sh +++ b/docker-makefiles/prep_configs.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -euo pipefail +#set -euo pipefail # -------- Helpers -------- die() { @@ -67,29 +67,21 @@ fi if [[ ! -f "${BASE_ENV}" ]]; then echo "Failed to locate file: ${BASE_ENV} — skipping LICENSE_KEY update" else - # If LICENSE_KEY is set in the environment (passed from Makefile), write it - # into the config file so docker compose env_file picks it up too. - if [[ -n "${LICENSE_KEY:-}" ]]; then - if grep -q '^LICENSE_KEY=' "${BASE_ENV}"; then - sedi "s|^LICENSE_KEY=.*|LICENSE_KEY=${LICENSE_KEY}|" "${BASE_ENV}" - echo "LICENSE_KEY written to ${BASE_ENV} from environment" - else - echo "LICENSE_KEY=${LICENSE_KEY}" >> "${BASE_ENV}" - echo "LICENSE_KEY appended to ${BASE_ENV} from environment" - fi + # Extract raw value (everything after LICENSE_KEY=) + CURRENT_LICENSE_KEY=$(sed -n 's/^LICENSE_KEY=//p' "${BASE_ENV}") + + # Strip trailing carriage return and any surrounding quotes + CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY%$'\r'}" + CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY#\"}" ; CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY%\"}" + CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY#\'}" ; CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY%\'}" + + if [[ -z "${CURRENT_LICENSE_KEY}" ]]; then + echo "No LICENSE_KEY found in ${BASE_ENV} — skipping" else - # Fall back: normalize whatever is already in the file - CURRENT_LICENSE_KEY=$(sed -n 's/^LICENSE_KEY=//p' "${BASE_ENV}") - CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY%$'\r'}" - CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY#\"}" ; CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY%\"}" - CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY#\'}" ; CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY%\'}" - if [[ -z "${CURRENT_LICENSE_KEY}" ]]; then - echo "No LICENSE_KEY in environment or ${BASE_ENV} — skipping" - else - UPDATED_LICENSE_KEY="${CURRENT_LICENSE_KEY//\'/\"}" - sedi "s|^LICENSE_KEY=.*|LICENSE_KEY=${UPDATED_LICENSE_KEY}|" "${BASE_ENV}" - echo "LICENSE_KEY updated in ${BASE_ENV}" - fi + # Replace any internal single quotes with double quotes + UPDATED_LICENSE_KEY="${CURRENT_LICENSE_KEY//\'/\"}" + sedi "s|^LICENSE_KEY=.*|LICENSE_KEY=${UPDATED_LICENSE_KEY}|" "${BASE_ENV}" + echo "LICENSE_KEY updated in ${BASE_ENV}" fi fi From dc9b5c9bc1885006dc03525d1f5581664b470199 Mon Sep 17 00:00:00 2001 From: Ori Shadmon Date: Tue, 19 May 2026 19:43:41 -0700 Subject: [PATCH 38/46] improve to support shell Signed-off-by: Ori Shadmon --- deploy.sh | 2 +- .../node_configs.env | 4 +- docker-makefiles/build_docker_compose.sh | 38 ++- .../docker-compose-template-base.yaml | 2 +- docker-makefiles/prep_configs.sh | 25 +- docker-makefiles/qualcomm/node_configs.env | 295 ++++++++++++++++++ 6 files changed, 354 insertions(+), 12 deletions(-) create mode 100644 docker-makefiles/qualcomm/node_configs.env diff --git a/deploy.sh b/deploy.sh index f1848483..26426095 100644 --- a/deploy.sh +++ b/deploy.sh @@ -2,7 +2,7 @@ # deploy.sh — AnyLog node lifecycle manager # Usage: bash deploy.sh [OPTIONS] # Run: bash deploy.sh help -set -euo pipefail +#set -euo pipefail # ────────────────────────────────────────────── # Defaults (override via environment or flags) diff --git a/docker-makefiles/anylog-standalone-operator/node_configs.env b/docker-makefiles/anylog-standalone-operator/node_configs.env index 754e1475..5ee04ab8 100644 --- a/docker-makefiles/anylog-standalone-operator/node_configs.env +++ b/docker-makefiles/anylog-standalone-operator/node_configs.env @@ -218,7 +218,7 @@ DNS_DOMAIN="" DNS="" # Bind settings - a true value binds to a specific IP and Port (a false value binds to all IPs) -TCP_BIND=false +TCP_BIND=true REST_BIND=false BROKER_BIND=false @@ -279,7 +279,7 @@ QUERY_POOL=6 #--- General --- # AnyLog License Key -LICENSE_KEY=cd554412f2a4ff540e3426a4f1fcb6d50ab8d9e1130e42957b0338ea45f4303fd160a551c5200b0321101abdc9eb44169e1b1c9d5a9492b16cc870619bb5199ae806a1dfadda93e00f02d30f3a5dabc0a6a7bb8f869e37fa565962eba55cb335397a66903d8fb3014ae1e4eea46c104cad8c787a0d5ae45efbf8728cf04cb805{company:Guest,expiration:2026-08-01,type:beta} +LICENSE_KEY=cd554412f2a4ff540e3426a4f1fcb6d50ab8d9e1130e42957b0338ea45f4303fd160a551c5200b0321101abdc9eb44169e1b1c9d5a9492b16cc870619bb5199ae806a1dfadda93e00f02d30f3a5dabc0a6a7bb8f869e37fa565962eba55cb335397a66903d8fb3014ae1e4eea46c104cad8c787a0d5ae45efbf8728cf04cb805{"company":"Guest","expiration":"2026-08-01","type":"beta"} #--- Database --- # Username for SQL database connection diff --git a/docker-makefiles/build_docker_compose.sh b/docker-makefiles/build_docker_compose.sh index 6ccbf99d..78239f4f 100644 --- a/docker-makefiles/build_docker_compose.sh +++ b/docker-makefiles/build_docker_compose.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -euo pipefail +#set -euo pipefail # -------- Helpers -------- die() { @@ -33,8 +33,8 @@ if [[ -f "docker-makefiles/${NODE_CONFIGS}/.env" ]] && \ ENV_FILE="docker-makefiles/${NODE_CONFIGS}/.env" BASE_ENV="docker-makefiles/${NODE_CONFIGS}/base_configs.env" elif [[ -f "docker-makefiles/${NODE_CONFIGS}/node_configs.env" ]]; then - ENV_FILE="docker-makefiles/${NODE_CONFIGS}/node_configs.env" - BASE_ENV="docker-makefiles/${NODE_CONFIGS}/node_configs.env" + ENV_FILE="docker-makefiles/${NODE_CONFIGS}/formatted_node_configs.env" + BASE_ENV="docker-makefiles/${NODE_CONFIGS}/formatted_node_configs.env" else die "Missing configuration file(s) for '${NODE_CONFIGS}', cannot continue" fi @@ -76,7 +76,7 @@ if [[ "$MULTI_FILE" == "true" ]]; then '/ env_file:/ {print; print " - " env; print " - " base; print " - " adv; next}1' \ "${COMPOSE_FILE}" > temp.yaml && mv temp.yaml "${COMPOSE_FILE}" else - awk -v cfg="../../docker-makefiles/${NODE_CONFIGS}/node_configs.env" \ + awk -v cfg="../../docker-makefiles/${NODE_CONFIGS}/formatted_node_configs.env" \ '/ env_file:/ {print; print " - " cfg; next}1' \ "${COMPOSE_FILE}" > temp.yaml && mv temp.yaml "${COMPOSE_FILE}" fi @@ -215,11 +215,37 @@ ${INJECT} done fi - # -------- Envsubst & Write Output -------- echo "Generating final docker-compose.yaml..." mkdir -p docker-makefiles/docker-compose-files OUTPUT_FILE="docker-makefiles/docker-compose-files/${NODE_CONFIGS}-docker-compose.yaml" envsubst < "${COMPOSE_FILE}" > "$OUTPUT_FILE" rm -rf ${COMPOSE_FILE} ${COMPOSE_FILE}.bak -echo "Saved: ${OUTPUT_FILE}" \ No newline at end of file +echo "Saved: ${OUTPUT_FILE}" + +# -------- Generate Read-Only Snapshot Copy -------- +# Filename: {formatted_node_name}.env (hyphens -> underscores, lowercased) +# Empty-value vars ( VAR="" ) are commented out in the copy. +FORMATTED_NODE_NAME=$(echo "${NODE_CONFIGS}" | tr '[:upper:]' '[:lower:]' | tr '-' '_') +SNAPSHOT_DIR="docker-makefiles/${NODE_CONFIGS}" +SNAPSHOT_FILE="${SNAPSHOT_DIR}/${FORMATTED_NODE_NAME}.env" + +echo "Generating read-only snapshot: ${SNAPSHOT_FILE}" + +# Collect config files that were used +if [[ "$MULTI_FILE" == "true" ]]; then + SNAPSHOT_SOURCES=("${ENV_FILE}" "${BASE_ENV}" "docker-makefiles/${NODE_CONFIGS}/advance_configs.env") +else + SNAPSHOT_SOURCES=("${ENV_FILE}") +fi + +{ + for src in "${SNAPSHOT_SOURCES[@]}"; do + echo "# ---- $(basename "${src}") ----" + sed -E 's/^([A-Za-z_][A-Za-z0-9_]*)=""(\s*(#.*)?)$/#\1=""\2/' "${src}" + echo "" + done +} > "${SNAPSHOT_FILE}" + +chmod 444 "${SNAPSHOT_FILE}" +echo "Snapshot saved (read-only): ${SNAPSHOT_FILE}" \ No newline at end of file diff --git a/docker-makefiles/docker-compose-template-base.yaml b/docker-makefiles/docker-compose-template-base.yaml index fb45bb0a..ed31a27d 100644 --- a/docker-makefiles/docker-compose-template-base.yaml +++ b/docker-makefiles/docker-compose-template-base.yaml @@ -20,7 +20,7 @@ services: - ${NODE_NAME}-local-scripts:/app/deployment-scripts ${NODE_NAME}: image: ${IMAGE}:${TAG} - user: "anylog:anylog" + user: root pid: "host" group_add: - ${DOCKER_GID} diff --git a/docker-makefiles/prep_configs.sh b/docker-makefiles/prep_configs.sh index 30d88654..5205509d 100644 --- a/docker-makefiles/prep_configs.sh +++ b/docker-makefiles/prep_configs.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -#set -euo pipefail +set -euo pipefail # -------- Helpers -------- die() { @@ -85,4 +85,25 @@ else fi fi -echo "Config update complete." +# -------- Step 3: Generate Read-Only Snapshot Copy -------- +# Filename: {formatted_node_name}.env (hyphens -> underscores, lowercased) +# Empty-value vars ( VAR="" ) are commented out in the copy. +FORMATTED_NODE_NAME=$(echo "${NODE_CONFIGS}" | tr '[:upper:]' '[:lower:]' | tr '-' '_') +SNAPSHOT_DIR="docker-makefiles/${NODE_CONFIGS}" +SNAPSHOT_FILE="${SNAPSHOT_DIR}/formatted_node_configs.env" + +echo "Generating read-only snapshot: ${SNAPSHOT_FILE}" + +# Concatenate all config files, comment out empty-value lines, write snapshot +{ + for cfg in "${CONFIG_FILES[@]}"; do + echo "# ---- $(basename "${cfg}") ----" + sed -E 's/^([A-Za-z_][A-Za-z0-9_]*)=""(\s*(#.*)?)$/#\1=""\2/' "${cfg}" + echo "" + done +} > "${SNAPSHOT_FILE}" + +chmod 444 "${SNAPSHOT_FILE}" +#echo "Snapshot saved (read-only): ${SNAPSHOT_FILE}" + +echo "Config update complete." \ No newline at end of file diff --git a/docker-makefiles/qualcomm/node_configs.env b/docker-makefiles/qualcomm/node_configs.env new file mode 100644 index 00000000..74aa88e5 --- /dev/null +++ b/docker-makefiles/qualcomm/node_configs.env @@ -0,0 +1,295 @@ +#==== .env ==== +# Basic deployment settings + +# Container image, deployment type +# -> prod: run AnyLog instance +# -> bash: run shell interface with AnyLog installed and params set +INIT_TYPE=prod + +# AnyLog image +IMAGE=anylogco/anylog-network + +# Github repo to use for configuring / deploying node +# The path can either be a Github URL and then the code will automatically clone +# the repo if needed OR a local path for deployment-scripts and then use that instead of cloning into the repositroy +DEPLOYMENTS_REPO=https://github.com/AnyLog-co/deployment-scripts +#DEPLOYMENTS_REPO=/anylog1/deployment-scripts +# Branch associated with git repo +DEPLOYMENTS_BRANCH="pre-develop" + +# specify the docker socket - to be used when deploying docker monitoring +DOCKER_SOCKET="/var/run/docker.sock" + +#==== basic ==== +# Core node configuration - required for all deployments +#--- General --- +# Configuration based deploymnet node deployment type +# Node Types: +# generic: sandbox with only network configured +# master: "blockchain" emulator +# operator: agent dedicated to storing data +# query: agent dedicated to querying data (enable system_query logical database) +# publisher: agent used to distribute data +# master-operator: a combination of master and operator on a single agent +# master-publisher: a combination of master and publisher on a single agent +NODE_TYPE=master-operator + +# Name of the AnyLog instance +NODE_NAME=anylog-standalone-operator +# Owner of the AnyLog instance +COMPANY_NAME=AnyLog Co. + +#--- Networking --- +# Port address used by AnyLog's TCP protocol to communicate with other nodes in the network +ANYLOG_SERVER_PORT=32148 +# Port address used by AnyLog's REST protocol +ANYLOG_REST_PORT=32149 +# Port value to be used as an MQTT broker, or some other third-party broker +ANYLOG_BROKER_PORT=32150 + +#--- Database --- +# Physical database type (sqlite, psql) +DB_TYPE=sqlite +# Database IP address +DB_IP=127.0.0.1 +# Database port number +DB_PORT=5432 +# Whether to set autocommit data +AUTOCOMMIT=true + +#--- NoSQL --- +# Enable external blob storage +BLOBS_STORAGE=false +# Storage backend type - Supported: mongo, s3, akave, minio +BLOB_STORAGE_TYPE=mongo +# Storage endpoint (MongoDB or object storage endpoint) +BLOB_STORAGE_IP=127.0.0.1 +# Storage port (used mainly for MongoDB) +BLOB_STORAGE_PORT=27017 + +#--- System Query --- +# Whether to start system_query logical database +SYSTEM_QUERY=true +# Database type used for system_query logical database +SYSTEM_QUERY_DB=sqlite +# Run system_query using in-memory SQLite. If set to false, will use pre-set database type +MEMORY=true + +#--- Blockchain --- +# TCP connection information for Master Node +LEDGER_CONN=127.0.0.1:32148 +# How often to sync from blockchain +BLOCKCHAIN_SYNC=60 second +# Source of where the blockchain data is coming from (master, optimism) +BLOCKCHAIN_SOURCE=master +# Where the local copy of the blockchain will be stored +BLOCKCHAIN_DESTINATION=file + +#--- Operator --- +# Owner of the cluster +CLUSTER_NAME=anylog-cluster1 +# Logical database name +DEFAULT_DBMS=new_company + +#==== southbound ==== +# Data ingestion protocols (MQTT, OPC-UA, video streaming, monitoring) +#--- Monitoring --- +# Whether to monitor the node or not +NODE_MONITORING=false +# Store monitoring in Operator node(s) +STORE_MONITORING=false +# Accept syslog data from local (Message broker required) +SYSLOG_MONITORING=false +# Docker Monitoring +DOCKER_MONITORING=false + +#--- Video Streaming --- +# Enable or disable video streaming. +# true = stream video, false = do not stream +ENABLE_VIDEO_STREAMING=false +# URL of the video to stream. Can be a YouTube link or other video source. +VIDEO_URL=https://www.youtube.com/watch?v=rnXIjl_Rzy4 +# Port on which the video streaming server will run. +VIDEO_PORT=32800 +# Name of the video stream or service identifier. +VIDEO_NAME=sfstream +# Enable or disable object detections (like YOLO detections) on the video. +# true = enable detections, false = disable +ENABLE_DETECTIONS=false +# Port on which the YOLO model server will run for object detection. +YOLO_MODEL_PORT=50051 + +#--- MQTT --- +# Whether to enable the default MQTT process +ENABLE_MQTT=false + +# IP address of MQTT broker +MQTT_BROKER=172.104.228.251 +# Port associated with MQTT broker +MQTT_PORT=1883 +# Whether to enable MQTT logging process +MQTT_LOG=false + +# Topic to get data for +MSG_TOPIC=rand-data +# Logical database name +MSG_DBMS=mydb +# Table where to store data +MSG_TABLE=bring [table] +# Timestamp column name +MSG_TIMESTAMP_COLUMN=bring [timestamp] +# Value column name +MSG_VALUE_COLUMN=bring [value] +# Column value type +MSG_VALUE_COLUMN_TYPE=float + +#--- OPC-UA --- +# Whether or not to enable to OPC-UA service +ENABLE_OPCUA=true +# OPC-UA URL address (ex. opcua.tcp:;//127.0.0.1:4840) +OPCUA_URL=opc.tcp://virtualfactory.proveit.services:4842/discovery +# Node information the root is located in (ex. ns=2;s=DataSet) +OPCUA_NODE="ns=2;s=Site1" +# How often to pull data from OPC-UA +OPCUA_FREQUENCY=5 + +#==== advanced ==== +# Fine-tuning for experienced users - safe to leave as defaults +#--- Directories --- +# AnyLog Root Path +ANYLOG_PATH=/app +# !local_scripts: ${ANYLOG_HOME}/deployment-scripts/scripts +LOCAL_SCRIPTS=/app/deployment-scripts +# !test_dir: ${ANYLOG_HOME}/deployment-scripts/test +TEST_DIR=/app/deployment-scripts/test + +#--- General --- +# hide blockchain policy - supports all node types except cluster / operator +IS_HIDDEN=false +# Whether to automatically run a local (or personalized) script at the end of the process +DEPLOY_LOCAL_SCRIPT=true +# Coordinates of the machine - used by Grafana to map the network +LOCATION="" +# Country where machine is located +COUNTRY="" +# State where machine is located +STATE="" +# City where machine is located +CITY="" + +#--- Networking --- +# Declare Policy name +CONFIG_NAME="" +# Whether to enable DNS resolution +ENABLE_DNS=false +# NIC type used to resolve the internal IP address (ex. eth0, lo) +NIC_TYPE="" +# Overlay IP address - if set, will replace local IP address when connecting to network +OVERLAY_IP="" +# External DNS resolution +EXTERNAL_DNS="" +# local DNS domain to append to hostname (ex. example.com) +DNS_DOMAIN="" +# Full path for machine's (local) DNS +DNS="" + +# Bind settings - a true value binds to a specific IP and Port (a false value binds to all IPs) +TCP_BIND=true +REST_BIND=false +BROKER_BIND=false + +# Number of concurrent threads per connection protocol +TCP_THREADS=6 +REST_THREADS=6 +BROKER_THREADS=6 + +# Timeout in seconds for REST requests (0 = continuous) +REST_TIMEOUT=30 + +#--- NoSQL --- +# Reuse identical blobs using file hash +BLOBS_REUSE=true +# Compress blobs before storage +BLOBS_COMPRESS=true + +# Folder for storing blobs locally +# Leave empty to use default application path +BLOBS_FOLDER="" + +#--- Object Store --- +# Logical bucket grouping +BUCKET_GROUP="" +# Bucket identifier +BUCKET_ID="" +# Cloud region +BUCKET_REGION="" + +#--- Operator --- +# Operator ID - autogenerated INT if not specified +# This is used to help keep a record of where data was originally stored +MEMBER="" +# Whether of not to enable HA against the cluster +ENABLE_HA=false +# How many days back to sync between nodes +START_DATE=30 +# How many threads to use in the operator process +OPERATOR_THREADS=3 + +#--- Sub processes --- +# Compress JSON and SQL file(s) backup +COMPRESS_FILE=true +# When data comes in write to database immediately, as opposed to waiting for a full buffer +WRITE_IMMEDIATE=true +# If buffer is not full, how long to wait until pushing data through +THRESHOLD_TIME=60 seconds +# Buffer size to reach, at which point data is pushed through +THRESHOLD_VOLUME=10KB +# Number of parallel queries +QUERY_POOL=6 + +#==== secrets ==== +# Sensitive credentials - do not commit to version control +# will be updated to use sercets logic for both docker and k8s +#--- General --- +# AnyLog License Key +LICENSE_KEY=cd554412f2a4ff540e3426a4f1fcb6d50ab8d9e1130e42957b0338ea45f4303fd160a551c5200b0321101abdc9eb44169e1b1c9d5a9492b16cc870619bb5199ae806a1dfadda93e00f02d30f3a5dabc0a6a7bb8f869e37fa565962eba55cb335397a66903d8fb3014ae1e4eea46c104cad8c787a0d5ae45efbf8728cf04cb805{"company":"Guest","expiration":"2026-08-01","type":"beta"} + +#--- Database --- +# Username for SQL database connection +DB_USER=admin +# Password correlated to database user +DB_PASSWD=passwd + +#--- NoSQL --- +# Optional DB authentication +BLOB_STORAGE_USER=admin +BLOB_STORAGE_PASSWORD=passwd + +#--- MQTT --- +# User associated with MQTT broker +MQTT_USER=anyloguser +# Password associated with MQTT user +MQTT_PASSWD=mqtt4AnyLog! + +#--- Object Store --- +# Access credentials +BUCKET_ACCESS_KEY="" +BUCKET_SECRET_KEY="" + +#==== remote-gui ==== +# Companion UI application - deployed alongside the AnyLog node +# Whether to enable Remote-GUI in deployment +ENABLE_REMOTE_GUI=false +# Remote-GUI version Tag +REMOTE_GUI_TAG=1.1.0 +# NIC type to use for Remote-GUI - if not set, uses 127.0.0.1 +REMOTE_GUI_NIC="" +# Remote GUI Frontend Port +REMOTE_GUI_FE=31800 +# Remote GUI Backend Port +REMOTE_GUI_BE=8080 +# Default connection to REST service (IP:Port) if not provided then autogenerated using `REMOTE_GUI_NIC` +REMOTE_CONN="" +# Grafana PATH to reference for Grafana connection +GRAFANA_URL="" + From d427d4aa9e40b873ef884e0669ce9009a1a28b4a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 21 May 2026 00:52:58 +0000 Subject: [PATCH 39/46] chore: update version for os-dev [skip ci] --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 2b83fe55..46396796 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = docker-compose -version = 1.4.2604 (os-dev - 71ebae [2026-05-13 23:06:34]) +version = 1.4.2604 (os-dev - dc9b5c [2026-05-21 00:52:58]) author = AnyLog Co. contact = info@anylog.co description = docker-compose configurations for AnyLog deployment From 30992755c76562ebd85e3a3a1fc01186869dc28f Mon Sep 17 00:00:00 2001 From: Ori Shadmon Date: Thu, 21 May 2026 13:21:50 -0700 Subject: [PATCH 40/46] support for comment out empty rather than use empty Signed-off-by: Ori Shadmon --- deploy.sh | 3 +- docker-makefiles/build_docker_compose.sh | 4 +- docker-makefiles/clean_configs.sh | 58 +++++++ docker-makefiles/prep_configs.sh | 196 +++++++++++------------ 4 files changed, 160 insertions(+), 101 deletions(-) create mode 100644 docker-makefiles/clean_configs.sh diff --git a/deploy.sh b/deploy.sh index 26426095..17349591 100644 --- a/deploy.sh +++ b/deploy.sh @@ -219,7 +219,6 @@ cmd_up() { fi } - cmd_down() { _check_configs _load_configs @@ -249,6 +248,7 @@ cmd_clean() { echo "Stopping + removing volumes: ${ANYLOG_TYPE}" ${DOCKER_COMPOSE_CMD} -f "${DOCKER_COMPOSE_FILE}" down -v fi + bash docker-makefiles/clean_configs.sh "${ANYLOG_TYPE}" } cmd_clean_all() { @@ -268,6 +268,7 @@ cmd_clean_all() { echo "Stopping + removing volumes + image: ${ANYLOG_TYPE}" ${DOCKER_COMPOSE_CMD} -f "${DOCKER_COMPOSE_FILE}" down -v --rmi all fi + bash docker-makefiles/clean_configs.sh "${ANYLOG_TYPE}" } cmd_logs() { diff --git a/docker-makefiles/build_docker_compose.sh b/docker-makefiles/build_docker_compose.sh index 78239f4f..be99af23 100644 --- a/docker-makefiles/build_docker_compose.sh +++ b/docker-makefiles/build_docker_compose.sh @@ -226,9 +226,9 @@ echo "Saved: ${OUTPUT_FILE}" # -------- Generate Read-Only Snapshot Copy -------- # Filename: {formatted_node_name}.env (hyphens -> underscores, lowercased) # Empty-value vars ( VAR="" ) are commented out in the copy. -FORMATTED_NODE_NAME=$(echo "${NODE_CONFIGS}" | tr '[:upper:]' '[:lower:]' | tr '-' '_') +#FORMATTED_NODE_NAME=$(echo "${NODE_CONFIGS}" | tr '[:upper:]' '[:lower:]' | tr '-' '_') SNAPSHOT_DIR="docker-makefiles/${NODE_CONFIGS}" -SNAPSHOT_FILE="${SNAPSHOT_DIR}/${FORMATTED_NODE_NAME}.env" +SNAPSHOT_FILE="${SNAPSHOT_DIR}/formatted_node_configs.env" echo "Generating read-only snapshot: ${SNAPSHOT_FILE}" diff --git a/docker-makefiles/clean_configs.sh b/docker-makefiles/clean_configs.sh new file mode 100644 index 00000000..fc6d69a4 --- /dev/null +++ b/docker-makefiles/clean_configs.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +set -euo pipefail + +# -------- Helpers -------- +die() { + echo "Error: $1" >&2 + exit "${2:-1}" +} + +sedi() { + local expr="$1" + shift + + if [ "$#" -eq 0 ]; then + echo "sedi: no files provided" >&2 + return 1 + fi + + if sed --version >/dev/null 2>&1; then + sed -i "$expr" "$@" + else + sed -i '' "$expr" "$@" + fi +} + +# -------- Args -------- +NODE_CONFIGS=${1:-anylog-generic} + +# -------- Locate Config Files -------- +if [[ -f "docker-makefiles/${NODE_CONFIGS}/.env" ]] && \ + [[ -f "docker-makefiles/${NODE_CONFIGS}/base_configs.env" ]] && \ + [[ -f "docker-makefiles/${NODE_CONFIGS}/advance_configs.env" ]]; then + CONFIG_FILES=( + "docker-makefiles/${NODE_CONFIGS}/.env" + "docker-makefiles/${NODE_CONFIGS}/base_configs.env" + "docker-makefiles/${NODE_CONFIGS}/advance_configs.env" + ) +elif [[ -f "docker-makefiles/${NODE_CONFIGS}/node_configs.env" ]]; then + CONFIG_FILES=("docker-makefiles/${NODE_CONFIGS}/node_configs.env") +else + die "Missing configuration file(s) for '${NODE_CONFIGS}', cannot continue" +fi + +# -------- Generate Read-Only Snapshot Copy -------- +# Filename: {formatted_node_name}.env (hyphens -> underscores, lowercased) +# Empty-value vars ( VAR="" ) are commented out in the copy. +FORMATTED_NODE_NAME=$(echo "${NODE_CONFIGS}" | tr '[:upper:]' '[:lower:]' | tr '-' '_') +SNAPSHOT_DIR="docker-makefiles/${NODE_CONFIGS}" +SNAPSHOT_FILE="${SNAPSHOT_DIR}/formatted_node_configs.env" + +# -------- Generate Docker Compose File -------- +DOCKER_DIR="docker-makefiles/docker-compose-files" +DOCKER_FILE="${DOCKER_DIR}/${NODE_CONFIGS}-docker-compose.yaml" + +# -------- Remove files -------- +rm -rf ${DOCKER_FILE} "${SNAPSHOT_FILE}" + +if [[ -z "$(ls -A ${DOCKER_DIR})" ]] ; then rm -rf "${DOCKER_DIR}" ; fi diff --git a/docker-makefiles/prep_configs.sh b/docker-makefiles/prep_configs.sh index 5205509d..ca59c28c 100644 --- a/docker-makefiles/prep_configs.sh +++ b/docker-makefiles/prep_configs.sh @@ -1,109 +1,109 @@ -#!/usr/bin/env bash -set -euo pipefail - -# -------- Helpers -------- -die() { - echo "Error: $1" >&2 - exit "${2:-1}" -} - -sedi() { - local expr="$1" - shift - - if [ "$#" -eq 0 ]; then - echo "sedi: no files provided" >&2 - return 1 + #!/usr/bin/env bash + set -euo pipefail + + # -------- Helpers -------- + die() { + echo "Error: $1" >&2 + exit "${2:-1}" + } + + sedi() { + local expr="$1" + shift + + if [ "$#" -eq 0 ]; then + echo "sedi: no files provided" >&2 + return 1 + fi + + if sed --version >/dev/null 2>&1; then + sed -i "$expr" "$@" + else + sed -i '' "$expr" "$@" + fi + } + + # -------- Args -------- + NODE_CONFIGS=${1:-anylog-generic} + + # -------- Locate Config Files -------- + if [[ -f "docker-makefiles/${NODE_CONFIGS}/.env" ]] && \ + [[ -f "docker-makefiles/${NODE_CONFIGS}/base_configs.env" ]] && \ + [[ -f "docker-makefiles/${NODE_CONFIGS}/advance_configs.env" ]]; then + CONFIG_FILES=( + "docker-makefiles/${NODE_CONFIGS}/.env" + "docker-makefiles/${NODE_CONFIGS}/base_configs.env" + "docker-makefiles/${NODE_CONFIGS}/advance_configs.env" + ) + elif [[ -f "docker-makefiles/${NODE_CONFIGS}/node_configs.env" ]]; then + CONFIG_FILES=("docker-makefiles/${NODE_CONFIGS}/node_configs.env") + else + die "Missing configuration file(s) for '${NODE_CONFIGS}', cannot continue" fi - if sed --version >/dev/null 2>&1; then - sed -i "$expr" "$@" + # -------- Step 1: Normalize Single-Quoted Values to Double-Quoted -------- + # Transforms: MY_VAR='XXX' -> MY_VAR="XXX" + # Leaves already-double-quoted, unquoted, and comment lines untouched. + normalize_quotes() { + local file="$1" + echo "Normalizing quotes in: ${file}" + sedi "s/=''/=\"\"/g" "${file}" + sedi -E "s/^([A-Za-z_][A-Za-z0-9_]*)='(.*)'/\1=\"\2\"/" "${file}" + } + + for cfg in "${CONFIG_FILES[@]}"; do + normalize_quotes "${cfg}" + done + + # -------- Step 2: Update LICENSE_KEY -------- + # Multi-file layout: LICENSE_KEY lives in base_configs.env (index 1). + # Single-file layout: LICENSE_KEY lives in node_configs.env (index 0). + if [[ ${#CONFIG_FILES[@]} -gt 1 ]]; then + BASE_ENV="${CONFIG_FILES[1]}" else - sed -i '' "$expr" "$@" + BASE_ENV="${CONFIG_FILES[0]}" fi -} - -# -------- Args -------- -NODE_CONFIGS=${1:-anylog-generic} - -# -------- Locate Config Files -------- -if [[ -f "docker-makefiles/${NODE_CONFIGS}/.env" ]] && \ - [[ -f "docker-makefiles/${NODE_CONFIGS}/base_configs.env" ]] && \ - [[ -f "docker-makefiles/${NODE_CONFIGS}/advance_configs.env" ]]; then - CONFIG_FILES=( - "docker-makefiles/${NODE_CONFIGS}/.env" - "docker-makefiles/${NODE_CONFIGS}/base_configs.env" - "docker-makefiles/${NODE_CONFIGS}/advance_configs.env" - ) -elif [[ -f "docker-makefiles/${NODE_CONFIGS}/node_configs.env" ]]; then - CONFIG_FILES=("docker-makefiles/${NODE_CONFIGS}/node_configs.env") -else - die "Missing configuration file(s) for '${NODE_CONFIGS}', cannot continue" -fi - -# -------- Step 1: Normalize Single-Quoted Values to Double-Quoted -------- -# Transforms: MY_VAR='XXX' -> MY_VAR="XXX" -# Leaves already-double-quoted, unquoted, and comment lines untouched. -normalize_quotes() { - local file="$1" - echo "Normalizing quotes in: ${file}" - sedi "s/=''/=\"\"/g" "${file}" - sedi -E "s/^([A-Za-z_][A-Za-z0-9_]*)='(.*)'/\1=\"\2\"/" "${file}" -} - -for cfg in "${CONFIG_FILES[@]}"; do - normalize_quotes "${cfg}" -done - -# -------- Step 2: Update LICENSE_KEY -------- -# Multi-file layout: LICENSE_KEY lives in base_configs.env (index 1). -# Single-file layout: LICENSE_KEY lives in node_configs.env (index 0). -if [[ ${#CONFIG_FILES[@]} -gt 1 ]]; then - BASE_ENV="${CONFIG_FILES[1]}" -else - BASE_ENV="${CONFIG_FILES[0]}" -fi - -if [[ ! -f "${BASE_ENV}" ]]; then - echo "Failed to locate file: ${BASE_ENV} — skipping LICENSE_KEY update" -else - # Extract raw value (everything after LICENSE_KEY=) - CURRENT_LICENSE_KEY=$(sed -n 's/^LICENSE_KEY=//p' "${BASE_ENV}") - - # Strip trailing carriage return and any surrounding quotes - CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY%$'\r'}" - CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY#\"}" ; CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY%\"}" - CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY#\'}" ; CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY%\'}" - - if [[ -z "${CURRENT_LICENSE_KEY}" ]]; then - echo "No LICENSE_KEY found in ${BASE_ENV} — skipping" + + if [[ ! -f "${BASE_ENV}" ]]; then + echo "Failed to locate file: ${BASE_ENV} — skipping LICENSE_KEY update" else - # Replace any internal single quotes with double quotes - UPDATED_LICENSE_KEY="${CURRENT_LICENSE_KEY//\'/\"}" - sedi "s|^LICENSE_KEY=.*|LICENSE_KEY=${UPDATED_LICENSE_KEY}|" "${BASE_ENV}" - echo "LICENSE_KEY updated in ${BASE_ENV}" + # Extract raw value (everything after LICENSE_KEY=) + CURRENT_LICENSE_KEY=$(sed -n 's/^LICENSE_KEY=//p' "${BASE_ENV}") + + # Strip trailing carriage return and any surrounding quotes + CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY%$'\r'}" + CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY#\"}" ; CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY%\"}" + CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY#\'}" ; CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY%\'}" + + if [[ -z "${CURRENT_LICENSE_KEY}" ]]; then + echo "No LICENSE_KEY found in ${BASE_ENV} — skipping" + else + # Replace any internal single quotes with double quotes + UPDATED_LICENSE_KEY="${CURRENT_LICENSE_KEY//\'/\"}" + sedi "s|^LICENSE_KEY=.*|LICENSE_KEY=${UPDATED_LICENSE_KEY}|" "${BASE_ENV}" + echo "LICENSE_KEY updated in ${BASE_ENV}" + fi fi -fi -# -------- Step 3: Generate Read-Only Snapshot Copy -------- -# Filename: {formatted_node_name}.env (hyphens -> underscores, lowercased) -# Empty-value vars ( VAR="" ) are commented out in the copy. -FORMATTED_NODE_NAME=$(echo "${NODE_CONFIGS}" | tr '[:upper:]' '[:lower:]' | tr '-' '_') -SNAPSHOT_DIR="docker-makefiles/${NODE_CONFIGS}" -SNAPSHOT_FILE="${SNAPSHOT_DIR}/formatted_node_configs.env" + # -------- Step 3: Generate Read-Only Snapshot Copy -------- + # Filename: {formatted_node_name}.env (hyphens -> underscores, lowercased) + # Empty-value vars ( VAR="" ) are commented out in the copy. + FORMATTED_NODE_NAME=$(echo "${NODE_CONFIGS}" | tr '[:upper:]' '[:lower:]' | tr '-' '_') + SNAPSHOT_DIR="docker-makefiles/${NODE_CONFIGS}" + SNAPSHOT_FILE="${SNAPSHOT_DIR}/formatted_node_configs.env" -echo "Generating read-only snapshot: ${SNAPSHOT_FILE}" + echo "Generating read-only snapshot: ${SNAPSHOT_FILE}" -# Concatenate all config files, comment out empty-value lines, write snapshot -{ - for cfg in "${CONFIG_FILES[@]}"; do - echo "# ---- $(basename "${cfg}") ----" - sed -E 's/^([A-Za-z_][A-Za-z0-9_]*)=""(\s*(#.*)?)$/#\1=""\2/' "${cfg}" - echo "" - done -} > "${SNAPSHOT_FILE}" + # Concatenate all config files, comment out empty-value lines, write snapshot + { + for cfg in "${CONFIG_FILES[@]}"; do + echo "# ---- $(basename "${cfg}") ----" + sed -E 's/^([A-Za-z_][A-Za-z0-9_]*)=""(\s*(#.*)?)$/#\1=""\2/' "${cfg}" + echo "" + done + } > "${SNAPSHOT_FILE}" -chmod 444 "${SNAPSHOT_FILE}" -#echo "Snapshot saved (read-only): ${SNAPSHOT_FILE}" + chmod 444 "${SNAPSHOT_FILE}" + #echo "Snapshot saved (read-only): ${SNAPSHOT_FILE}" -echo "Config update complete." \ No newline at end of file + echo "Config update complete." \ No newline at end of file From c512124bbdeb702a545bdd53ed6b4c05affc4bdb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 21 May 2026 20:22:11 +0000 Subject: [PATCH 41/46] chore: update version for os-dev [skip ci] --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 46396796..677a9a64 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = docker-compose -version = 1.4.2604 (os-dev - dc9b5c [2026-05-21 00:52:58]) +version = 1.4.2604 (os-dev - fcdc60 [2026-05-21 20:22:11]) author = AnyLog Co. contact = info@anylog.co description = docker-compose configurations for AnyLog deployment From 901f20ada9f0b65f42f16456a5cc7a8360a990dd Mon Sep 17 00:00:00 2001 From: Ori Shadmon Date: Thu, 21 May 2026 13:34:37 -0700 Subject: [PATCH 42/46] prep configs call from build_Docker_ompose.sh Signed-off-by: Ori Shadmon --- deploy.sh | 2 +- docker-makefiles/build_docker_compose.sh | 53 +++++++++++++----------- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/deploy.sh b/deploy.sh index 17349591..d897938c 100644 --- a/deploy.sh +++ b/deploy.sh @@ -162,7 +162,7 @@ cmd_dry_run() { _load_configs if [[ "$IS_MANUAL" == "false" ]]; then echo "Dry Run ${ANYLOG_TYPE} - ${NODE_NAME}" - bash docker-makefiles/prep_configs.sh "${ANYLOG_TYPE}" +# bash docker-makefiles/prep_configs.sh "${ANYLOG_TYPE}" bash docker-makefiles/build_docker_compose.sh "${ANYLOG_TYPE}" "${TAG}" elif [[ "${IS_MANUAL}" == "true" ]]; then local single_file="docker-makefiles/${ANYLOG_TYPE}/node_configs.env" diff --git a/docker-makefiles/build_docker_compose.sh b/docker-makefiles/build_docker_compose.sh index be99af23..a997c867 100644 --- a/docker-makefiles/build_docker_compose.sh +++ b/docker-makefiles/build_docker_compose.sh @@ -39,6 +39,10 @@ else die "Missing configuration file(s) for '${NODE_CONFIGS}', cannot continue" fi +# -------- Generate Read-Only Snapshot Copy -------- +bash docker-makefiles/prep_configs.sh "${ANYLOG_TYPE}" + + # -------- Load Configs -------- export IMAGE=$(grep -m1 '^IMAGE=' "$ENV_FILE" | cut -d= -f2- | tr -d '"\r') export ENABLE_REMOTE_GUI=$(grep -m1 '^ENABLE_REMOTE_GUI=' "$ENV_FILE" | cut -d= -f2- | tr -d '"\r') @@ -223,29 +227,30 @@ envsubst < "${COMPOSE_FILE}" > "$OUTPUT_FILE" rm -rf ${COMPOSE_FILE} ${COMPOSE_FILE}.bak echo "Saved: ${OUTPUT_FILE}" -# -------- Generate Read-Only Snapshot Copy -------- -# Filename: {formatted_node_name}.env (hyphens -> underscores, lowercased) -# Empty-value vars ( VAR="" ) are commented out in the copy. -#FORMATTED_NODE_NAME=$(echo "${NODE_CONFIGS}" | tr '[:upper:]' '[:lower:]' | tr '-' '_') -SNAPSHOT_DIR="docker-makefiles/${NODE_CONFIGS}" -SNAPSHOT_FILE="${SNAPSHOT_DIR}/formatted_node_configs.env" - -echo "Generating read-only snapshot: ${SNAPSHOT_FILE}" -# Collect config files that were used -if [[ "$MULTI_FILE" == "true" ]]; then - SNAPSHOT_SOURCES=("${ENV_FILE}" "${BASE_ENV}" "docker-makefiles/${NODE_CONFIGS}/advance_configs.env") -else - SNAPSHOT_SOURCES=("${ENV_FILE}") -fi - -{ - for src in "${SNAPSHOT_SOURCES[@]}"; do - echo "# ---- $(basename "${src}") ----" - sed -E 's/^([A-Za-z_][A-Za-z0-9_]*)=""(\s*(#.*)?)$/#\1=""\2/' "${src}" - echo "" - done -} > "${SNAPSHOT_FILE}" -chmod 444 "${SNAPSHOT_FILE}" -echo "Snapshot saved (read-only): ${SNAPSHOT_FILE}" \ No newline at end of file +## Filename: {formatted_node_name}.env (hyphens -> underscores, lowercased) +## Empty-value vars ( VAR="" ) are commented out in the copy. +##FORMATTED_NODE_NAME=$(echo "${NODE_CONFIGS}" | tr '[:upper:]' '[:lower:]' | tr '-' '_') +#SNAPSHOT_DIR="docker-makefiles/${NODE_CONFIGS}" +#SNAPSHOT_FILE="${SNAPSHOT_DIR}/formatted_node_configs.env" +# +#echo "Generating read-only snapshot: ${SNAPSHOT_FILE}" +# +## Collect config files that were used +#if [[ "$MULTI_FILE" == "true" ]]; then +# SNAPSHOT_SOURCES=("${ENV_FILE}" "${BASE_ENV}" "docker-makefiles/${NODE_CONFIGS}/advance_configs.env") +#else +# SNAPSHOT_SOURCES=("${ENV_FILE}") +#fi +# +#{ +# for src in "${SNAPSHOT_SOURCES[@]}"; do +# echo "# ---- $(basename "${src}") ----" +# sed -E 's/^([A-Za-z_][A-Za-z0-9_]*)=""(\s*(#.*)?)$/#\1=""\2/' "${src}" +# echo "" +# done +#} > "${SNAPSHOT_FILE}" +# +#chmod 444 "${SNAPSHOT_FILE}" +#echo "Snapshot saved (read-only): ${SNAPSHOT_FILE}" \ No newline at end of file From d81c889ca5d1b41da49828482ce567b2fa295356 Mon Sep 17 00:00:00 2001 From: Ori Shadmon Date: Thu, 21 May 2026 13:38:54 -0700 Subject: [PATCH 43/46] prep configs call from build_Docker_ompose.sh Signed-off-by: Ori Shadmon --- docker-makefiles/clean_configs.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker-makefiles/clean_configs.sh b/docker-makefiles/clean_configs.sh index fc6d69a4..80fa31ef 100644 --- a/docker-makefiles/clean_configs.sh +++ b/docker-makefiles/clean_configs.sh @@ -55,4 +55,6 @@ DOCKER_FILE="${DOCKER_DIR}/${NODE_CONFIGS}-docker-compose.yaml" # -------- Remove files -------- rm -rf ${DOCKER_FILE} "${SNAPSHOT_FILE}" -if [[ -z "$(ls -A ${DOCKER_DIR})" ]] ; then rm -rf "${DOCKER_DIR}" ; fi +if [[ -d "${DOCKER_DIR}" && -z "$(ls -A "${DOCKER_DIR}")" ]]; then + rm -rf "${DOCKER_DIR}" +fi \ No newline at end of file From ca11b5670e777f393f3c8b0b1edb405c8e45a92f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 21 May 2026 20:39:12 +0000 Subject: [PATCH 44/46] chore: update version for os-dev [skip ci] --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 677a9a64..563149c1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = docker-compose -version = 1.4.2604 (os-dev - fcdc60 [2026-05-21 20:22:11]) +version = 1.4.2604 (os-dev - 12ea42 [2026-05-21 20:39:12]) author = AnyLog Co. contact = info@anylog.co description = docker-compose configurations for AnyLog deployment From 95e8ebcf79a029a8dbf875ffce88445907055495 Mon Sep 17 00:00:00 2001 From: Ori Shadmon Date: Thu, 21 May 2026 14:08:09 -0700 Subject: [PATCH 45/46] comment out 'set -euo pipefail' Signed-off-by: Ori Shadmon --- docker-makefiles/clean_configs.sh | 2 +- docker-makefiles/prep_configs.sh | 196 +++++++++++++++--------------- 2 files changed, 99 insertions(+), 99 deletions(-) diff --git a/docker-makefiles/clean_configs.sh b/docker-makefiles/clean_configs.sh index 80fa31ef..7f569693 100644 --- a/docker-makefiles/clean_configs.sh +++ b/docker-makefiles/clean_configs.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -euo pipefail +#set -euo pipefail # -------- Helpers -------- die() { diff --git a/docker-makefiles/prep_configs.sh b/docker-makefiles/prep_configs.sh index ca59c28c..4a861716 100644 --- a/docker-makefiles/prep_configs.sh +++ b/docker-makefiles/prep_configs.sh @@ -1,109 +1,109 @@ - #!/usr/bin/env bash - set -euo pipefail - - # -------- Helpers -------- - die() { - echo "Error: $1" >&2 - exit "${2:-1}" - } - - sedi() { - local expr="$1" - shift - - if [ "$#" -eq 0 ]; then - echo "sedi: no files provided" >&2 - return 1 - fi - - if sed --version >/dev/null 2>&1; then - sed -i "$expr" "$@" - else - sed -i '' "$expr" "$@" - fi - } - - # -------- Args -------- - NODE_CONFIGS=${1:-anylog-generic} - - # -------- Locate Config Files -------- - if [[ -f "docker-makefiles/${NODE_CONFIGS}/.env" ]] && \ - [[ -f "docker-makefiles/${NODE_CONFIGS}/base_configs.env" ]] && \ - [[ -f "docker-makefiles/${NODE_CONFIGS}/advance_configs.env" ]]; then - CONFIG_FILES=( - "docker-makefiles/${NODE_CONFIGS}/.env" - "docker-makefiles/${NODE_CONFIGS}/base_configs.env" - "docker-makefiles/${NODE_CONFIGS}/advance_configs.env" - ) - elif [[ -f "docker-makefiles/${NODE_CONFIGS}/node_configs.env" ]]; then - CONFIG_FILES=("docker-makefiles/${NODE_CONFIGS}/node_configs.env") - else - die "Missing configuration file(s) for '${NODE_CONFIGS}', cannot continue" +#!/usr/bin/env bash +#set -euo pipefail + +# -------- Helpers -------- +die() { + echo "Error: $1" >&2 + exit "${2:-1}" +} + +sedi() { + local expr="$1" + shift + + if [ "$#" -eq 0 ]; then + echo "sedi: no files provided" >&2 + return 1 fi - # -------- Step 1: Normalize Single-Quoted Values to Double-Quoted -------- - # Transforms: MY_VAR='XXX' -> MY_VAR="XXX" - # Leaves already-double-quoted, unquoted, and comment lines untouched. - normalize_quotes() { - local file="$1" - echo "Normalizing quotes in: ${file}" - sedi "s/=''/=\"\"/g" "${file}" - sedi -E "s/^([A-Za-z_][A-Za-z0-9_]*)='(.*)'/\1=\"\2\"/" "${file}" - } - - for cfg in "${CONFIG_FILES[@]}"; do - normalize_quotes "${cfg}" - done - - # -------- Step 2: Update LICENSE_KEY -------- - # Multi-file layout: LICENSE_KEY lives in base_configs.env (index 1). - # Single-file layout: LICENSE_KEY lives in node_configs.env (index 0). - if [[ ${#CONFIG_FILES[@]} -gt 1 ]]; then - BASE_ENV="${CONFIG_FILES[1]}" + if sed --version >/dev/null 2>&1; then + sed -i "$expr" "$@" else - BASE_ENV="${CONFIG_FILES[0]}" + sed -i '' "$expr" "$@" fi - - if [[ ! -f "${BASE_ENV}" ]]; then - echo "Failed to locate file: ${BASE_ENV} — skipping LICENSE_KEY update" +} + +# -------- Args -------- +NODE_CONFIGS=${1:-anylog-generic} + +# -------- Locate Config Files -------- +if [[ -f "docker-makefiles/${NODE_CONFIGS}/.env" ]] && \ + [[ -f "docker-makefiles/${NODE_CONFIGS}/base_configs.env" ]] && \ + [[ -f "docker-makefiles/${NODE_CONFIGS}/advance_configs.env" ]]; then + CONFIG_FILES=( + "docker-makefiles/${NODE_CONFIGS}/.env" + "docker-makefiles/${NODE_CONFIGS}/base_configs.env" + "docker-makefiles/${NODE_CONFIGS}/advance_configs.env" + ) +elif [[ -f "docker-makefiles/${NODE_CONFIGS}/node_configs.env" ]]; then + CONFIG_FILES=("docker-makefiles/${NODE_CONFIGS}/node_configs.env") +else + die "Missing configuration file(s) for '${NODE_CONFIGS}', cannot continue" +fi + +# -------- Step 1: Normalize Single-Quoted Values to Double-Quoted -------- +# Transforms: MY_VAR='XXX' -> MY_VAR="XXX" +# Leaves already-double-quoted, unquoted, and comment lines untouched. +normalize_quotes() { + local file="$1" + echo "Normalizing quotes in: ${file}" + sedi "s/=''/=\"\"/g" "${file}" + sedi -E "s/^([A-Za-z_][A-Za-z0-9_]*)='(.*)'/\1=\"\2\"/" "${file}" +} + +for cfg in "${CONFIG_FILES[@]}"; do + normalize_quotes "${cfg}" +done + +# -------- Step 2: Update LICENSE_KEY -------- +# Multi-file layout: LICENSE_KEY lives in base_configs.env (index 1). +# Single-file layout: LICENSE_KEY lives in node_configs.env (index 0). +if [[ ${#CONFIG_FILES[@]} -gt 1 ]]; then + BASE_ENV="${CONFIG_FILES[1]}" +else + BASE_ENV="${CONFIG_FILES[0]}" +fi + +if [[ ! -f "${BASE_ENV}" ]]; then + echo "Failed to locate file: ${BASE_ENV} — skipping LICENSE_KEY update" +else + # Extract raw value (everything after LICENSE_KEY=) + CURRENT_LICENSE_KEY=$(sed -n 's/^LICENSE_KEY=//p' "${BASE_ENV}") + + # Strip trailing carriage return and any surrounding quotes + CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY%$'\r'}" + CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY#\"}" ; CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY%\"}" + CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY#\'}" ; CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY%\'}" + + if [[ -z "${CURRENT_LICENSE_KEY}" ]]; then + echo "No LICENSE_KEY found in ${BASE_ENV} — skipping" else - # Extract raw value (everything after LICENSE_KEY=) - CURRENT_LICENSE_KEY=$(sed -n 's/^LICENSE_KEY=//p' "${BASE_ENV}") - - # Strip trailing carriage return and any surrounding quotes - CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY%$'\r'}" - CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY#\"}" ; CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY%\"}" - CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY#\'}" ; CURRENT_LICENSE_KEY="${CURRENT_LICENSE_KEY%\'}" - - if [[ -z "${CURRENT_LICENSE_KEY}" ]]; then - echo "No LICENSE_KEY found in ${BASE_ENV} — skipping" - else - # Replace any internal single quotes with double quotes - UPDATED_LICENSE_KEY="${CURRENT_LICENSE_KEY//\'/\"}" - sedi "s|^LICENSE_KEY=.*|LICENSE_KEY=${UPDATED_LICENSE_KEY}|" "${BASE_ENV}" - echo "LICENSE_KEY updated in ${BASE_ENV}" - fi + # Replace any internal single quotes with double quotes + UPDATED_LICENSE_KEY="${CURRENT_LICENSE_KEY//\'/\"}" + sedi "s|^LICENSE_KEY=.*|LICENSE_KEY=${UPDATED_LICENSE_KEY}|" "${BASE_ENV}" + echo "LICENSE_KEY updated in ${BASE_ENV}" fi +fi - # -------- Step 3: Generate Read-Only Snapshot Copy -------- - # Filename: {formatted_node_name}.env (hyphens -> underscores, lowercased) - # Empty-value vars ( VAR="" ) are commented out in the copy. - FORMATTED_NODE_NAME=$(echo "${NODE_CONFIGS}" | tr '[:upper:]' '[:lower:]' | tr '-' '_') - SNAPSHOT_DIR="docker-makefiles/${NODE_CONFIGS}" - SNAPSHOT_FILE="${SNAPSHOT_DIR}/formatted_node_configs.env" +# -------- Step 3: Generate Read-Only Snapshot Copy -------- +# Filename: {formatted_node_name}.env (hyphens -> underscores, lowercased) +# Empty-value vars ( VAR="" ) are commented out in the copy. +FORMATTED_NODE_NAME=$(echo "${NODE_CONFIGS}" | tr '[:upper:]' '[:lower:]' | tr '-' '_') +SNAPSHOT_DIR="docker-makefiles/${NODE_CONFIGS}" +SNAPSHOT_FILE="${SNAPSHOT_DIR}/formatted_node_configs.env" - echo "Generating read-only snapshot: ${SNAPSHOT_FILE}" +echo "Generating read-only snapshot: ${SNAPSHOT_FILE}" - # Concatenate all config files, comment out empty-value lines, write snapshot - { - for cfg in "${CONFIG_FILES[@]}"; do - echo "# ---- $(basename "${cfg}") ----" - sed -E 's/^([A-Za-z_][A-Za-z0-9_]*)=""(\s*(#.*)?)$/#\1=""\2/' "${cfg}" - echo "" - done - } > "${SNAPSHOT_FILE}" +# Concatenate all config files, comment out empty-value lines, write snapshot +{ + for cfg in "${CONFIG_FILES[@]}"; do + echo "# ---- $(basename "${cfg}") ----" + sed -E 's/^([A-Za-z_][A-Za-z0-9_]*)=""(\s*(#.*)?)$/#\1=""\2/' "${cfg}" + echo "" + done +} > "${SNAPSHOT_FILE}" - chmod 444 "${SNAPSHOT_FILE}" - #echo "Snapshot saved (read-only): ${SNAPSHOT_FILE}" +chmod 444 "${SNAPSHOT_FILE}" +#echo "Snapshot saved (read-only): ${SNAPSHOT_FILE}" - echo "Config update complete." \ No newline at end of file +echo "Config update complete." \ No newline at end of file From 4c9ba0c1e00e7d433fa75e7f20a8193cc8809468 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 21 May 2026 21:08:29 +0000 Subject: [PATCH 46/46] chore: update version for os-dev [skip ci] --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 563149c1..3da1afae 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = docker-compose -version = 1.4.2604 (os-dev - 12ea42 [2026-05-21 20:39:12]) +version = 1.4.2604 (os-dev - e4a077 [2026-05-21 21:08:29]) author = AnyLog Co. contact = info@anylog.co description = docker-compose configurations for AnyLog deployment