Feat/update hornet

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.4
+FROM golang:1.23.9
 
 # install requisite debian components
 RUN apt-get update && apt-get install -y rsync
@@ -16,40 +16,17 @@ RUN echo "{\n}" > ~/.project8_authentications.json && \
     mkdir /data/hot && \
     mkdir /data/warm && \
     mkdir /data/cold && \
-    # modify hornet config to be purely local
-    sed -e '9s/true/false/' \
-        # disable slack
-        -e '19s/true/false/' \
-        # use the directory names above for storage
-        -e 's@"dir": "/data"@"dir": "/data/hot"@' \
-        -e 's@"/warm-data"@"/data/warm"@' \
-        # no amqp connection so disable sending file info
-        -e 's@"send-file-info": true@"send-file-info": false@' \
-        # shipper config to just local move
-        -e 's@"/remote-data"@"/data/cold"@' \
-        -e '101s/,//' \
-        -e '102,103d' \
-        # remove base-paths
-        -e '59,60d' \
-        # remove estra dirs from watcher
-        -e '30,34d' \
-        -e '29s/,//' \
-        /go/src/github.com/project8/hornet/examples/hornet_config.json > /go/hornet_config.json
-
-# use go to install golang deps
-RUN go get github.com/streadway/amqp \
-           github.com/ugorji/go/codec \
-           github.com/op/go-logging \
-           golang.org/x/exp/inotify \
-           github.com/kardianos/osext \
-           code.google.com/p/go-uuid/uuid \
-           github.com/spf13/viper
+    cp /go/src/github.com/project8/hornet/examples/hornet_config_local.json /go/hornet_config.json
 
 # This next is a hack, it requires you to have first done ``cp ~/.project8_authentications project8_authentications``
 # There is probably a data-volumes based solution that cleans this up
 
-RUN cd /go/src/github.com/project8/hornet && make remove_older_describe_go
+RUN cd /go/src/github.com/project8/hornet && go install
 
-RUN go install github.com/project8/hornet
+# Create some test files for transfer
+
+RUN touch /data/hot/test1.Setup
+RUN touch /data/hot/test2.Setup
+RUN touch /data/hot/test3.Setup
 
 CMD ["hornet", "-config", "hornet_config.json"]

config.sh

@@ -0,0 +1,12 @@
+sed -e '9s/true/false/' \
+-e '19s/true/false/' \
+-e 's@"dir": "/data"@"dir": "/data/hot"@' \
+-e 's@"/warm-data"@"/data/warm"@' \
+-e 's@"send-file-info": true@"send-file-info": false@' \
+-e 's@"/remote-data"@"/data/cold"@' \
+-e '101s/,//' \
+-e '102,103d' \
+-e '59,60d' \
+-e '30,34d' \
+-e '29s/,//' \
+./examples/hornet_config.json > ./hornet_config.json

examples/hornet_config.json

@@ -106,4 +106,4 @@
         "hostname": "my.server",
         "username": "aphysicist"
     }
-}
+}

examples/hornet_config_local.json

@@ -0,0 +1,105 @@
+{
+    "scheduler":
+    {
+        "queue-size": 100,
+        "summary-interval": "1m"
+    },
+
+    "logger":
+    {
+        "level": "NOTICE"
+    },
+
+    "amqp":
+    {
+        "active": false,
+        "use-auth": "true",
+        "broker": "my.server",
+        "port": 5672,
+        "exchange": "requests",
+        "queue": "hornet"
+     },
+
+    "slack":
+    {
+        "active": false,
+        "username": "hornet",
+        "alerts-channel": "#p8_alerts",
+        "notices-channel": "#p8_notices"
+    },
+
+    "watcher":
+    {
+        "active": true,
+        "dir": "/data",
+        "ignore-dirs":
+        [
+            "lost+found",
+	    "warm"
+        ],
+        "file-wait-time": "5s"
+    },
+
+    "classifier":
+    {
+        "types":
+        [
+            {
+                "name": "egg",
+                "match-regexp": "runid(?P<run_id>[0-9]*)_(?P<fname_other>[A-Za-z0-9_]*).egg",
+                "do-hash": true
+            },
+            {
+                "name": "rsa-mat",
+                "match-regexp": "runid(?P<run_id>[0-9]*)_(?P<fname_other>[A-Za-z0-9_]*).mat",
+                "do-hash": false
+            },
+            {
+                "name": "rsa-setup",
+                "match-extension": "Setup",
+                "do-hash": false
+            }
+        ],
+        "base-paths":
+        [
+            "/some/special/path",
+            "/another/path"
+        ],
+        "send-file-info": false,
+        "send-to": "hornet.print-message",
+        "wait-for-sender": 2,
+        "max-jobs": 25
+    },
+
+    "mover":
+    {
+        "dest-dir": "/data/warm"
+    },
+
+    "workers":
+    {
+        "n-workers": 5,
+        "jobs":
+        [
+            {
+                "name": "proc-egg",
+                "file-type": "egg",
+                "command": "echo \"here's an egg file\""
+            },
+            {
+                "name": "proc-rsa-mat",
+                "file-type": "rsa-mat",
+                "command": "echo \"here's an RSA MAT file\""
+            }
+        ]
+    },
+
+    "shipper":
+    {
+        "active": false,
+        "n-shippers": 1,
+        "dest-dir": "/data/cold",
+        "hostname": "my.server",
+        "username": "aphysicist"
+    }
+}

examples/hornet_config_sm3723.json

@@ -0,0 +1,99 @@
+{
+    "scheduler":
+    {
+        "queue-size": 100,
+        "summary-interval": "1m"
+    },
+
+    "logger":
+    {
+        "level": "NOTICE"
+    },
+
+    "amqp":
+    {
+        "active": false,
+        "use-auth": "false",
+        "broker": "localhost",
+        "port": 5672,
+        "exchange": "requests",
+        "queue": "hornet"
+     },
+
+    "slack":
+    {
+        "active": false,
+        "username": "hornet",
+        "alerts-channel": "#p8_alerts",
+        "notices-channel": "#p8_notices"
+    },
+
+    "watcher":
+    {
+        "active": true,
+        "dir": "/home/sm3723/project8/tutorial",
+        "ignore-dirs":
+        [
+            "warm-data"
+        ],
+        "file-wait-time": "5s"
+    },
+
+    "classifier":
+    {
+        "types":
+        [
+            {
+                "name": "egg",
+                "match-extension": "egg",
+                "do-hash": true
+            },
+            {
+                "name": "rsa-mat",
+                "match-regexp": "runid(?P<run_id>[0-9]*)_(?P<fname_other>[A-Za-z0-9_]*).mat",
+                "do-hash": true
+            },
+            {
+                "name": "rsa-setup",
+                "match-extension": "Setup",
+                "do-hash": false
+            }
+        ],
+        "send-file-info": false,
+        "send-to": "hornet.print-message",
+        "wait-for-sender": 2,
+        "max-jobs": 25
+    },
+
+    "mover":
+    {
+        "dest-dir": "/home/sm3723/project8/tutorial/warm-data"
+    },
+
+    "workers":
+    {
+        "n-workers": 5,
+        "jobs":
+        [
+            {
+                "name": "proc-egg",
+                "file-type": "egg",
+                "command": "echo \"here's an egg file\""
+            },
+            {
+                "name": "proc-rsa-mat",
+                "file-type": "rsa-mat",
+                "command": "echo \"here's an RSA MAT file\""
+            }
+        ]
+    },
+
+    "shipper":
+    {
+        "active": false,
+        "n-shippers": 1,
+        "dest-dir": "/remote-data",
+        "hostname": "my.server",
+        "username": "aphysicist"
+    }
+}

go.mod

@@ -0,0 +1,32 @@
+module github.com/project8/hornet
+
+go 1.23.9
+
+require (
+	github.com/carlmjohnson/versioninfo v0.22.5
+	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0
+	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
+	github.com/pborman/uuid v1.2.1
+	github.com/spf13/viper v1.20.1
+	github.com/streadway/amqp v1.1.0
+	github.com/ugorji/go/codec v1.3.0
+	gopkg.in/fsnotify.v1 v1.4.7
+)
+
+require (
+	github.com/fsnotify/fsnotify v1.8.0 // indirect
+	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
+	github.com/sagikazarmark/locafero v0.7.0 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.12.0 // indirect
+	github.com/spf13/cast v1.7.1 // indirect
+	github.com/spf13/pflag v1.0.6 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
+	go.uber.org/multierr v1.9.0 // indirect
+	golang.org/x/sys v0.29.0 // indirect
+	golang.org/x/text v0.21.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

go.sum

@@ -0,0 +1,69 @@
+github.com/carlmjohnson/versioninfo v0.22.5 h1:O00sjOLUAFxYQjlN/bzYTuZiS0y6fWDQjMRvwtKgwwc=
+github.com/carlmjohnson/versioninfo v0.22.5/go.mod h1:QT9mph3wcVfISUKd0i9sZfVrPviHuSF+cUtLjm2WSf8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
+github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
+github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
+github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
+github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss=
+github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA=
+github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
+github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
+github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw=
+github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
+github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
+github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo=
+github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k=
+github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
+github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
+github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs=
+github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4=
+github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y=
+github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
+github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
+github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4=
+github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4=
+github.com/streadway/amqp v1.1.0 h1:py12iX8XSyI7aN/3dUT8DFIDJazNJsVJdxNVEpnQTZM=
+github.com/streadway/amqp v1.1.0/go.mod h1:WYSrTEYHOXHd0nwFeUXAe2G2hRnQT+deZJJf88uS9Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
+github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
+github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA=
+github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4=
+go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
+go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
+go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
+golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
+golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=