diff --git a/.gitignore b/.gitignore index d80b0b3bdf..fc852d945b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ vuls .vscode *.txt *.json -*.sqlite3 +*.sqlite3* *.db tags .gitmodules @@ -12,3 +12,4 @@ vendor/ log/ results/ *config.toml +!setup/docker/* diff --git a/ISSUE_TEMPLATE b/ISSUE_TEMPLATE new file mode 100644 index 0000000000..51f9b478f7 --- /dev/null +++ b/ISSUE_TEMPLATE @@ -0,0 +1,36 @@ + +# Environment + +## Vuls + +Hash : ____ + +To check the commit hash of HEAD +$ vuls -v + +or +$ cd $GOPATH/src/github.com/future-architect/vuls +$ git rev-parse --short HEAD + +## OS +- Target Server: Write here +- Vuls Server: Write here + +## Go +- Go version: here + +# Current Output + +Please re-run the command using ```-debug``` and provide the output below. + +# Addition Details + +Can you also please fill in each of the remaining sections. + +## Expected Behavior + +## Actual Behavior + +## Steps to reproduce the behaviour + + diff --git a/Makefile b/Makefile index d29699d7cf..702e6dcb04 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,9 @@ .PHONY: \ + glide \ + deps \ + update \ + build \ + install \ all \ vendor \ lint \ @@ -12,12 +17,27 @@ SRCS = $(shell git ls-files '*.go') PKGS = ./. ./config ./models ./report ./cveapi ./scan ./util ./commands ./cache +VERSION := $(shell git describe --tags --abbrev=0) +REVISION := $(shell git rev-parse --short HEAD) +LDFLAGS := -X 'main.version=$(VERSION)' \ + -X 'main.revision=$(REVISION)' -all: test +glide: + go get github.com/Masterminds/glide + +deps: glide + glide install + +update: glide + glide update -# vendor: -# @ go get -v github.com/mjibson/party -# party -d external -c -u +build: main.go deps + go build -ldflags "$(LDFLAGS)" -o vuls $< + +install: main.go deps + go install -ldflags "$(LDFLAGS)" + +all: test lint: @ go get -v github.com/golang/lint/golint diff --git a/README.fr.md b/README.fr.md index e8ae519fd8..69a34e6c41 100644 --- a/README.fr.md +++ b/README.fr.md @@ -107,14 +107,14 @@ Vuls requiert l'installation des paquets suivants : - sqlite - git - gcc -- go v1.6 +- go v1.7.1 or later - https://golang.org/doc/install ```bash $ ssh ec2-user@52.100.100.100 -i ~/.ssh/private.pem $ sudo yum -y install sqlite git gcc -$ wget https://storage.googleapis.com/golang/go1.6.linux-amd64.tar.gz -$ sudo tar -C /usr/local -xzf go1.6.linux-amd64.tar.gz +$ wget https://storage.googleapis.com/golang/go1.7.1.linux-amd64.tar.gz +$ sudo tar -C /usr/local -xzf go1.7.1.linux-amd64.tar.gz $ mkdir $HOME/go ``` Ajoutez les lignes suivantes dans /etc/profile.d/goenv.sh @@ -201,7 +201,7 @@ Summary Unspecified vulnerability in the Java SE and Java SE Embedded co NVD https://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2016-0494 MITRE https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-0494 CVE Details http://www.cvedetails.com/cve/CVE-2016-0494 -CVSS Claculator https://nvd.nist.gov/cvss/v2-calculator?name=CVE-2016-0494&vector=(AV:N/AC:L/Au:N/C:C/I:C/A:C) +CVSS Calculator https://nvd.nist.gov/cvss/v2-calculator?name=CVE-2016-0494&vector=(AV:N/AC:L/Au:N/C:C/I:C/A:C) RHEL-CVE https://access.redhat.com/security/cve/CVE-2016-0494 ALAS-2016-643 https://alas.aws.amazon.com/ALAS-2016-643.html Package/CPE java-1.7.0-openjdk-1.7.0.91-2.6.2.2.63.amzn1 -> java-1.7.0-openjdk-1:1.7.0.95-2.6.4.0.65.amzn1 diff --git a/README.ja.md b/README.ja.md index 8df0f87a7a..bc4b5da9ff 100644 --- a/README.ja.md +++ b/README.ja.md @@ -46,6 +46,8 @@ Vulsは上に挙げた手動運用での課題を解決するツールであり - CPEに登録されているソフトウェアが対象 - エージェントレスアーキテクチャ - スキャン対象サーバにSSH接続可能なマシン1台にセットアップするだけで動作 +- 非破壊スキャン(SSHでコマンド発行するだけ) +- AWSでの脆弱性/侵入テスト事前申請は必要なし - 設定ファイルのテンプレート自動生成 - CIDRを指定してサーバを自動検出、設定ファイルのテンプレートを生成 - EmailやSlackで通知可能（日本語でのレポートも可能） @@ -65,7 +67,6 @@ Vulsは上に挙げた手動運用での課題を解決するツールであり Vulsのセットアップは以下の３パターンがある - Dockerコンテナ上にセットアップ -Docker Composeを用いて少ないコマンドでセットアップ可能 see https://github.com/future-architect/vuls/tree/master/setup/docker [日本語README](https://github.com/future-architect/vuls/blob/master/setup/docker/README.ja.md) - Chefでセットアップ @@ -121,17 +122,17 @@ VulsはSSHパスワード認証をサポートしていない。SSH公開鍵鍵 Vulsセットアップに必要な以下のソフトウェアをインストールする。 -- SQLite3 -- git v2 +- SQLite3 or MySQL +- git - gcc -- go v1.6 +- go v1.7.1 or later - https://golang.org/doc/install ```bash $ ssh ec2-user@52.100.100.100 -i ~/.ssh/private.pem $ sudo yum -y install sqlite git gcc -$ wget https://storage.googleapis.com/golang/go1.6.linux-amd64.tar.gz -$ sudo tar -C /usr/local -xzf go1.6.linux-amd64.tar.gz +$ wget https://storage.googleapis.com/golang/go1.7.1.linux-amd64.tar.gz +$ sudo tar -C /usr/local -xzf go1.7.1.linux-amd64.tar.gz $ mkdir $HOME/go ``` /etc/profile.d/goenv.sh を作成し、下記を追加する。 @@ -149,26 +150,19 @@ $ source /etc/profile.d/goenv.sh ## Step4. Deploy [go-cve-dictionary](https://github.com/kotakanbe/go-cve-dictionary) -go get - ```bash $ sudo mkdir /var/log/vuls $ sudo chown ec2-user /var/log/vuls $ sudo chmod 700 /var/log/vuls -$ go get github.com/kotakanbe/go-cve-dictionary +$ +$ mkdir -p $GOPATH/src/github.com/kotakanbe +$ cd $GOPATH/src/github.com/kotakanbe +$ git https://github.com/kotakanbe/go-cve-dictionary.git +$ cd go-cve-dictionary +$ make install ``` +バイナリは、`$GOPATH/bin`いかに生成される -go-cve-dictionaryを既にインストール済みでupdateしたい場合は - -```bash -$ go get -u github.com/kotakanbe/go-cve-dictionary -``` - -で可能である。 - -go getでエラーが発生した場合は、以下の点を確認する。 -- Gitのバージョンがv2以降か？ -- Go依存パッケージの問題でgo getに失敗する場合は [deploying with glide](https://github.com/future-architect/vuls/blob/master/README.md#deploy-with-glide) を試す。 NVDから脆弱性データベースを取得する。 環境によって異なるが、AWS上では10分程度かかる。 @@ -183,10 +177,12 @@ $ ls -alh cve.sqlite3 ## Step5. Deploy Vuls 新規にターミナルを起動し、先ほど作成したEC2にSSH接続する。 - -go get ``` -$ go get github.com/future-architect/vuls +$ mkdir -p $GOPATH/src/github.com/future-architect +$ cd $GOPATH/src/github.com/future-architect +$ git clone https://github.com/future-architect/vuls.git +$ cd vuls +$ make install ``` vulsを既にインストール済みでupdateしたい場合は @@ -229,7 +225,7 @@ $ vuls prepare ## Step8. Start Scanning ``` -$ vuls scan -cve-dictionary-dbpath=$PWD/cve.sqlite3 +$ vuls scan -cve-dictionary-dbpath=$PWD/cve.sqlite3 -report-json INFO[0000] Start scanning (config: /home/ec2-user/config.toml) INFO[0000] Start scanning INFO[0000] config: /home/ec2-user/config.toml @@ -256,7 +252,7 @@ Summary Unspecified vulnerability in the Java SE and Java SE Embedded co NVD https://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2016-0494 MITRE https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-0494 CVE Details http://www.cvedetails.com/cve/CVE-2016-0494 -CVSS Claculator https://nvd.nist.gov/cvss/v2-calculator?name=CVE-2016-0494&vector=(AV:N/AC:L/Au:N/C:C/I:C/A:C) +CVSS Calculator https://nvd.nist.gov/cvss/v2-calculator?name=CVE-2016-0494&vector=(AV:N/AC:L/Au:N/C:C/I:C/A:C) RHEL-CVE https://access.redhat.com/security/cve/CVE-2016-0494 ALAS-2016-643 https://alas.aws.amazon.com/ALAS-2016-643.html Package/CPE java-1.7.0-openjdk-1.7.0.91-2.6.2.2.63.amzn1 -> java-1.7.0-openjdk-1:1.7.0.95-2.6.4.0.65.amzn1 @@ -375,7 +371,7 @@ notifyUsers = ["@username"] [mail] smtpAddr = "smtp.gmail.com" -smtpPort = "465" +smtpPort = "587" user = "username" password = "password" from = "from@address.com" @@ -428,7 +424,7 @@ host = "172.31.4.82" notifyUsers = ["@username"] ``` - - hookURL : Incomming webhook's URL + - hookURL : Incoming webhook's URL - channel : channel name. channelに`${servername}`を指定すると、結果レポートをサーバごとに別チャネルにすることが出来る。 以下のサンプルでは、`#server1`チャネルと`#server2`チャネルに送信される。スキャン前にチャネルを作成する必要がある。 @@ -456,7 +452,7 @@ host = "172.31.4.82" ``` [mail] smtpAddr = "smtp.gmail.com" - smtpPort = "465" + smtpPort = "587" user = "username" password = "password" from = "from@address.com" @@ -475,6 +471,7 @@ host = "172.31.4.82" # "cpe:/a:rubyonrails:ruby_on_rails:4.2.1", #] #containers = ["${running}"] + #ignoreCves = ["CVE-2016-6313"] #optional = [ # ["key", "value"], #] @@ -494,6 +491,7 @@ host = "172.31.4.82" # "cpe:/a:rubyonrails:ruby_on_rails:4.2.1", #] #containers = ["${running}"] + #ignoreCves = ["CVE-2016-6314"] #optional = [ # ["key", "value"], #] @@ -508,6 +506,7 @@ host = "172.31.4.82" - keyPath: SSH private key path - cpeNames: see [Usage: Scan vulnerability of non-OS package](https://github.com/future-architect/vuls/blob/master/README.ja.md#usage-scan-vulnerability-of-non-os-package) - containers: see [Usage: Scan Docker containers](https://github.com/future-architect/vuls/blob/master/README.ja.md#usage-scan-docker-containers) + - ignoreCves: CVE IDs that will not be reported. But output to JSON file. - optional: JSONレポートに含めたい追加情報 @@ -563,7 +562,7 @@ vuls ALL=(root) NOPASSWD: /usr/bin/apt-get, /usr/bin/apt-cache # Usage: Prepare -Prepareサブコマンドは、Vuls内部で利用する以下のパッケージをスキャン対象サーバにインストーする。 +Prepareサブコマンドは、Vuls内部で利用する以下のパッケージをスキャン対象サーバにインストールする。 | Distribution| Release | Requirements | |:------------|-------------------:|:-------------| @@ -578,17 +577,22 @@ Prepareサブコマンドは、Vuls内部で利用する以下のパッケージ ``` $ vuls prepare -help -prepare - [-config=/path/to/config.toml] [-debug] +prepare: + prepare + [-config=/path/to/config.toml] [-ask-key-password] - [SERVER]... + [-debug] + [-ssh-external] + [SERVER]... -ask-key-password Ask ssh privatekey password before scanning -config string /path/to/toml (default "$PWD/config.toml") -debug debug mode + -ssh-external + Use external ssh command. Default: Use the Go native implementation ``` ---- @@ -602,7 +606,8 @@ scan: [-lang=en|ja] [-config=/path/to/config.toml] [-results-dir=/path/to/results] - [-cve-dictionary-dbpath=/path/to/cve.sqlite3] + [-cve-dictionary-dbtype=sqlite3|mysql] + [-cve-dictionary-dbpath=/path/to/cve.sqlite3 or mysql connection string] [-cve-dictionary-url=http://127.0.0.1:1323] [-cache-dbpath=/path/to/cache.db] [-cvss-over=7] @@ -649,7 +654,9 @@ scan: -containers-only Scan concontainers Only. Default: Scan both of hosts and containers -cve-dictionary-dbpath string - /path/to/sqlite3 (For get cve detail from cve.sqlite3) + /path/to/sqlite3 (For get cve detail from cve.sqlite3) + -cve-dictionary-dbtype string + DB type for fetching CVE dictionary (sqlite3 or mysql) (default "sqlite3") -cve-dictionary-url string http://CVE.Dictionary (default "http://127.0.0.1:1323") -cvss-over float @@ -788,6 +795,43 @@ $ vuls scan \ -azure-container=vuls ``` +## Example: IgnoreCves + +Slack, Mail, テキスト出力しないくないCVE IDがある場合は、設定ファイルに定義することでレポートされなくなる。 +ただ、JSONファイルには以下のように出力される。 + +- config.toml +```toml +[default] +ignoreCves = ["CVE-2016-6313"] + +[servers.bsd] +host = "192.168.11.11" +user = "kanbe" +ignoreCves = ["CVE-2016-6314"] +``` + +- bsd.json +```json +[ + { + "ServerName": "bsd", + "Family": "FreeBSD", + "Release": "10.3-RELEASE", + "IgnoredCves" : [ + "CveDetail" : { + "CVE-2016-6313", + ... + }, + "CveDetail" : { + "CVE-2016-6314", + ... + } + ] + } +] +``` + ## Example: Add optional key-value pairs to JSON 追加情報をJSONに含めることができる。 @@ -828,6 +872,14 @@ optional = [ ] ``` +## Example: Use MySQL as a DB storage back-end + +``` +$ vuls scan \ + -cve-dictionary-dbtype=mysql \ + -cve-dictionary-dbpath="user:pass@tcp(localhost:3306)/dbname?parseTime=true" +``` + ---- # Usage: Scan vulnerability of non-OS package @@ -855,6 +907,31 @@ Vulsは、[CPE](https://nvd.nist.gov/cpe.cfm)に登録されているソフト "cpe:/a:rubyonrails:ruby_on_rails:4.2.1", ] ``` + + +# Usage: Integrate with OWASP Dependency Check to Automatic update when the libraries are updated (Experimental) +[OWASP Dependency check](https://www.owasp.org/index.php/OWASP_Dependency_Check) は、プログラミング言語のライブラリを特定し（CPEを推測）、公開済みの脆弱性を検知するツール。 + +VulsとDependency Checkを連携させる方法は以下 +- Dependency Checkを、--format=XMLをつけて実行する +- そのXMLをconfig.toml内で以下のように定義する + + ``` + [servers] + + [servers.172-31-4-82] + host = "172.31.4.82" + user = "ec2-user" + keyPath = "/home/username/.ssh/id_rsa" + dependencyCheckXMLPath = "/tmp/dependency-check-report.xml" + ``` + +VulsとDependency Checkの連携すると以下の利点がある +- ライブラリを更新した場合に、config.tomlのCPEの定義を変更しなくても良い +- Vulsの機能でSlack, Emailで通知可能 +- 日本語のレポートが可能 + - Dependency Checkは日本語レポートに対応していない + # Usage: Scan Docker containers @@ -909,14 +986,14 @@ tui: ``` -Key binding is bellow. +Key binding is below. | key | | |:-----------------|:-------|:------| | TAB | move cursor among the panes | | Arrow up/down | move cursor to up/down | -| Ctrl+j, Ctrl+k | move cursor to up/donw | -| Ctrl+u, Ctrl+d | page up/donw | +| Ctrl+j, Ctrl+k | move cursor to up/down | +| Ctrl+u, Ctrl+d | page up/down | For details, see https://github.com/future-architect/vuls/blob/master/report/tui.go @@ -962,89 +1039,14 @@ $ vuls scan -cve-dictionary-url=http://192.168.0.1:1323 # Usage: Update NVD Data -``` -$ go-cve-dictionary fetchnvd -h -fetchnvd: - fetchnvd - [-last2y] - [-dbpath=/path/to/cve.sqlite3] - [-debug] - [-debug-sql] - - -dbpath string - /path/to/sqlite3 (default "$PWD/cve.sqlite3") - -debug - debug mode - -debug-sql - SQL debug mode - -last2y - Refresh NVD data in the last two years. -``` - -- Fetch data of the entire period - -``` -$ for i in {2002..2016}; do go-cve-dictionary fetchnvd -years $i; done -``` - -- Fetch data in the last 2 years - -``` -$ go-cve-dictionary fetchnvd -last2y -``` +see [go-cve-dictionary#usage-fetch-nvd-data](https://github.com/kotakanbe/go-cve-dictionary#usage-fetch-nvd-data) ---- # レポートの日本語化 -- JVNから日本語の脆弱性情報を取得 - ``` - $ go-cve-dictionary fetchjvn -h - fetchjvn: - fetchjvn - [-latest] - [-last2y] - [-years] 1998 1999 ... - [-dbpath=$PWD/cve.sqlite3] - [-http-proxy=http://192.168.0.1:8080] - [-debug] - [-debug-sql] - - -dbpath string - /path/to/sqlite3 (default "$PWD/cve.sqlite3") - -debug - debug mode - -debug-sql - SQL debug mode - -http-proxy string - http://proxy-url:port (default: empty) - -last2y - Refresh JVN data in the last two years. - -latest - Refresh JVN data for latest. - -years - Refresh JVN data of specific years. +see [go-cve-dictionary#usage-fetch-jvn-data](https://github.com/kotakanbe/go-cve-dictionary#usage-fetch-jvn-data) - ``` - -- すべての期間の脆弱性情報を取得(10分未満) - ``` - $ for i in {1998..2016}; do ./go-cve-dictionary fetchjvn -years $i; done - ``` - -- 2年分の情報を取得 - ``` - $ go-cve-dictionary fetchjvn -last2y - ``` - -- 最新情報のみ取得 - ``` - $ go-cve-dictionary fetchjvn -latest - ``` - -- 脆弱性情報の自動アップデート -Cronなどのジョブスケジューラを用いて実現可能。 --latestオプションを指定して夜間の日次実行を推奨。 ## fetchnvd, fetchjvnの実行順序の注意 @@ -1075,45 +1077,23 @@ slack, emailは日本語対応済み TUIは日本語表示未対応 ---- -# Deploy With Glide - -If an error occurred while go get, try deploying with glide. -- Install [Glide](https://github.com/Masterminds/glide) -- Deploy go-cve-dictionary -``` -$ go get -d github.com/kotakanbe/go-cve-dictionary -$ cd $GOPATH/src/github.com/kotakanbe/go-cve-dictionary -$ glide install -$ go install -``` -- Deploy vuls -``` -$ go get -d github.com/future-architect/vuls -$ cd $GOPATH/src/github.com/future-architect/vuls -$ glide install -$ go install -``` -- The binaries are created under $GOPARH/bin - ----- - # Update Vuls With Glide - Update go-cve-dictionary -If the DB schema was changed, please specify new SQLite3 DB file. +If the DB schema was changed, please specify new SQLite3 or MySQL DB file. ``` $ cd $GOPATH/src/github.com/kotakanbe/go-cve-dictionary $ git pull -$ glide install -$ go install +$ mv vendor /tmp/foo +$ make install ``` - Update vuls ``` $ cd $GOPATH/src/github.com/future-architect/vuls $ git pull -$ glide install -$ go install +$ mv vendor /tmp/bar +$ make install ``` - バイナリファイルは`$GOPARH/bin`以下に作成される @@ -1210,7 +1190,7 @@ Please see [CHANGELOG](https://github.com/future-architect/vuls/blob/master/CHAN ---- -# Licence +# License Please see [LICENSE](https://github.com/future-architect/vuls/blob/master/LICENSE). diff --git a/README.md b/README.md index d1e5819cdc..75eb206561 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,8 @@ Vuls is a tool created to solve the problems listed above. It has the following - Support software registered in CPE - Agentless architecture - User is required to only setup one machine that is connected to other target servers via SSH +- Nondestructive testing +- Pre-authorization is not necessary　before scanning on AWS - Auto generation of configuration file template - Auto detection of servers set using CIDR, generate configuration file template - Email and Slack notification is possible (supports Japanese language) @@ -128,17 +130,17 @@ And also, SUDO with password is not supported for security reasons. So you have Vuls requires the following packages. -- SQLite3 -- git v2 +- SQLite3 or MySQL +- git - gcc -- go v1.6 +- go v1.7.1 or later - https://golang.org/doc/install ```bash $ ssh ec2-user@52.100.100.100 -i ~/.ssh/private.pem $ sudo yum -y install sqlite git gcc -$ wget https://storage.googleapis.com/golang/go1.6.linux-amd64.tar.gz -$ sudo tar -C /usr/local -xzf go1.6.linux-amd64.tar.gz +$ wget https://storage.googleapis.com/golang/go1.7.1.linux-amd64.tar.gz +$ sudo tar -C /usr/local -xzf go1.7.1.linux-amd64.tar.gz $ mkdir $HOME/go ``` Add these lines into /etc/profile.d/goenv.sh @@ -156,18 +158,18 @@ $ source /etc/profile.d/goenv.sh ## Step4. Deploy [go-cve-dictionary](https://github.com/kotakanbe/go-cve-dictionary) -go get - ```bash $ sudo mkdir /var/log/vuls $ sudo chown ec2-user /var/log/vuls $ sudo chmod 700 /var/log/vuls -$ go get github.com/kotakanbe/go-cve-dictionary +$ +$ mkdir -p $GOPATH/src/github.com/kotakanbe +$ cd $GOPATH/src/github.com/kotakanbe +$ git clone https://github.com/kotakanbe/go-cve-dictionary.git +$ cd go-cve-dictionary +$ make install ``` - -If an error occurred while go get, check the following points. -- Update Git -- try [deploying with glide](https://github.com/future-architect/vuls/blob/master/README.md#deploy-with-glide). +The binary was built under `$GOPARH/bin` Fetch vulnerability data from NVD. It takes about 10 minutes (on AWS). @@ -183,14 +185,14 @@ $ ls -alh cve.sqlite3 Launch a new terminal and SSH to the ec2 instance. -go get ``` -$ go get github.com/future-architect/vuls +$ mkdir -p $GOPATH/src/github.com/future-architect +$ cd $GOPATH/src/github.com/future-architect +$ git clone https://github.com/future-architect/vuls.git +$ cd vuls +$ make install ``` - -If an error occurred while go get, check the following points. -- Update Git -- try [deploying with glide](https://github.com/future-architect/vuls/blob/master/README.md#deploy-with-glide). +The binary was built under `$GOPARH/bin` ## Step6. Config @@ -220,7 +222,7 @@ see [Usage: Prepare](https://github.com/future-architect/vuls#usage-prepare) ## Step8. Start Scanning ``` -$ vuls scan -cve-dictionary-dbpath=$PWD/cve.sqlite3 +$ vuls scan -cve-dictionary-dbpath=$PWD/cve.sqlite3 -report-json INFO[0000] Start scanning (config: /home/ec2-user/config.toml) INFO[0000] Start scanning INFO[0000] config: /home/ec2-user/config.toml @@ -247,7 +249,7 @@ Summary Unspecified vulnerability in the Java SE and Java SE Embedded co NVD https://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2016-0494 MITRE https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-0494 CVE Details http://www.cvedetails.com/cve/CVE-2016-0494 -CVSS Claculator https://nvd.nist.gov/cvss/v2-calculator?name=CVE-2016-0494&vector=(AV:N/AC:L/Au:N/C:C/I:C/A:C) +CVSS Calculator https://nvd.nist.gov/cvss/v2-calculator?name=CVE-2016-0494&vector=(AV:N/AC:L/Au:N/C:C/I:C/A:C) RHEL-CVE https://access.redhat.com/security/cve/CVE-2016-0494 ALAS-2016-643 https://alas.aws.amazon.com/ALAS-2016-643.html Package/CPE java-1.7.0-openjdk-1.7.0.91-2.6.2.2.63.amzn1 -> java-1.7.0-openjdk-1:1.7.0.95-2.6.4.0.65.amzn1 @@ -282,7 +284,7 @@ see https://github.com/future-architect/vuls/tree/master/setup/docker ![Vuls-Architecture](img/vuls-architecture.png) ## [go-cve-dictinary](https://github.com/kotakanbe/go-cve-dictionary) -- Fetch vulnerability information from NVD and JVN(Japanese), then insert into SQLite3. +- Fetch vulnerability information from NVD and JVN(Japanese), then insert into SQLite3 or MySQL. ## Scanning Flow ![Vuls-Scan-Flow](img/vuls-scan-flow.png) @@ -373,7 +375,7 @@ notifyUsers = ["@username"] [mail] smtpAddr = "smtp.gmail.com" -smtpPort = "465" +smtpPort = "587" user = "username" password = "password" from = "from@address.com" @@ -389,6 +391,7 @@ subjectPrefix = "[vuls]" # "cpe:/a:rubyonrails:ruby_on_rails:4.2.1", #] #containers = ["${running}"] +#ignoreCves = ["CVE-2016-6313"] #optional = [ # ["key", "value"], #] @@ -404,6 +407,7 @@ host = "172.31.4.82" # "cpe:/a:rubyonrails:ruby_on_rails:4.2.1", #] #containers = ["${running}"] +#ignoreCves = ["CVE-2016-6313"] #optional = [ # ["key", "value"], #] @@ -426,7 +430,7 @@ You can customize your configuration using this template. notifyUsers = ["@username"] ``` - - hookURL : Incomming webhook's URL + - hookURL : Incoming webhook's URL - channel : channel name. If you set `${servername}` to channel, the report will be sent to each channel. In the following example, the report will be sent to the `#server1` and `#server2`. @@ -457,7 +461,7 @@ You can customize your configuration using this template. ``` [mail] smtpAddr = "smtp.gmail.com" - smtpPort = "465" + smtpPort = "587" user = "username" password = "password" from = "from@address.com" @@ -476,6 +480,7 @@ You can customize your configuration using this template. # "cpe:/a:rubyonrails:ruby_on_rails:4.2.1", #] #containers = ["${running}"] + #ignoreCves = ["CVE-2016-6313"] #optional = [ # ["key", "value"], #] @@ -495,6 +500,7 @@ You can customize your configuration using this template. # "cpe:/a:rubyonrails:ruby_on_rails:4.2.1", #] #containers = ["${running}"] + #ignoreCves = ["CVE-2016-6314"] #optional = [ # ["key", "value"], #] @@ -508,6 +514,7 @@ You can customize your configuration using this template. - keyPath: SSH private key path - cpeNames: see [Usage: Scan vulnerability of non-OS package](https://github.com/future-architect/vuls#usage-scan-vulnerability-of-non-os-package) - containers: see [Usage: Scan Docker containers](https://github.com/future-architect/vuls#usage-scan-docker-containers) + - ignoreCves: CVE IDs that will not be reported. But output to JSON file. - optional: Add additional information to JSON report. Vuls supports two types of SSH. One is native go implementation. The other is external SSH command. For details, see [-ssh-external option](https://github.com/future-architect/vuls#-ssh-external-option) @@ -576,17 +583,22 @@ Prepare subcommand installs required packages on each server. ``` $ vuls prepare -help -prepare - [-config=/path/to/config.toml] [-debug] +prepare: + prepare + [-config=/path/to/config.toml] [-ask-key-password] - [SERVER]... + [-debug] + [-ssh-external] + [SERVER]... -ask-key-password Ask ssh privatekey password before scanning -config string /path/to/toml (default "$PWD/config.toml") -debug debug mode + -ssh-external + Use external ssh command. Default: Use the Go native implementation ``` ---- @@ -601,7 +613,8 @@ scan: [-lang=en|ja] [-config=/path/to/config.toml] [-results-dir=/path/to/results] - [-cve-dictionary-dbpath=/path/to/cve.sqlite3] + [-cve-dictionary-dbtype=sqlite3|mysql] + [-cve-dictionary-dbpath=/path/to/cve.sqlite3 or mysql connection string] [-cve-dictionary-url=http://127.0.0.1:1323] [-cache-dbpath=/path/to/cache.db] [-cvss-over=7] @@ -648,7 +661,9 @@ scan: -containers-only Scan concontainers Only. Default: Scan both of hosts and containers -cve-dictionary-dbpath string - /path/to/sqlite3 (For get cve detail from cve.sqlite3) + /path/to/sqlite3 (For get cve detail from cve.sqlite3) + -cve-dictionary-dbtype string + DB type for fetching CVE dictionary (sqlite3 or mysql) (default "sqlite3") -cve-dictionary-url string http://CVE.Dictionary (default "http://127.0.0.1:1323") -cvss-over float @@ -683,11 +698,11 @@ scan: Vuls supports different types of SSH. -By Defaut, using a native Go implementation from crypto/ssh. +By Default, using a native Go implementation from crypto/ssh. This is useful in situations where you may not have access to traditional UNIX tools. To use external ssh command, specify this option. -This is useful If you want to use ProxyCommand or chiper algorithm of SSH that is not supported by native go implementation. +This is useful If you want to use ProxyCommand or cipher algorithm of SSH that is not supported by native go implementation. Don't forget to add below line to /etc/sudoers on the target servers. (username: vuls) ``` Defaults:vuls !requiretty @@ -704,7 +719,7 @@ Defaults:vuls !requiretty ## -report-json , -report-text option At the end of the scan, scan results will be available in the `$PWD/result/current/` directory. -`all.(json|txt)` includes the scan results of all servres and `servername.(json|txt)` includes the scan result of the server. +`all.(json|txt)` includes the scan results of all servers and `servername.(json|txt)` includes the scan result of the server. ## Example: Scan all servers defined in config file ``` @@ -716,7 +731,7 @@ $ vuls scan \ -cve-dictionary-dbpath=$PWD/cve.sqlite3 ``` With this sample command, it will .. -- Ask SSH key passsword before scanning +- Ask SSH key password before scanning - Scan all servers defined in config file - Send scan results to slack and email - Only Report CVEs that CVSS score is over 7 @@ -780,6 +795,43 @@ $ vuls scan \ -azure-container=vuls ``` +## Example: IgnoreCves + +Define ignoreCves in config if you don't want to report(slack, mail, text...) specific CVE IDs. But these ignoreCves will be output to JSON file like below. + +- config.toml +```toml +[default] +ignoreCves = ["CVE-2016-6313"] + +[servers.bsd] +host = "192.168.11.11" +user = "kanbe" +ignoreCves = ["CVE-2016-6314"] +``` + +- bsd.json +```json +[ + { + "ServerName": "bsd", + "Family": "FreeBSD", + "Release": "10.3-RELEASE", + "IgnoredCves" : [ + "CveDetail" : { + "CVE-2016-6313", + ... + }, + "CveDetail" : { + "CVE-2016-6314", + ... + } + ] + } +] +``` + + ## Example: Add optional key-value pairs to JSON Optional key-value can be outputted to JSON. @@ -820,9 +872,17 @@ optional = [ ] ``` +## Example: Use MySQL as a DB storage back-end + +``` +$ vuls scan \ + -cve-dictionary-dbtype=mysql \ + -cve-dictionary-dbpath="user:pass@tcp(localhost:3306)/dbname?parseTime=true" +``` + ---- -# Usage: Scan vulnerability of non-OS package +# Usage: Scan vulnerabilites of non-OS packages It is possible to detect vulnerabilities in non-OS packages, such as something you compiled by yourself, language libraries and frameworks, that have been registered in the [CPE](https://nvd.nist.gov/cpe.cfm). @@ -831,10 +891,10 @@ It is possible to detect vulnerabilities in non-OS packages, such as something y **Check CPE Naming Format: 2.2** - [go-cpe-dictionary](https://github.com/kotakanbe/go-cpe-dictionary) is a good choice for geeks. - You can search a CPE name by the application name incremenally. + You can search a CPE name by the application name incrementally. - Configuration -To detect the vulnerbility of Ruby on Rails v4.2.1, cpeNames needs to be set in the servers section. +To detect the vulnerability of Ruby on Rails v4.2.1, cpeNames needs to be set in the servers section. ``` [servers] @@ -846,10 +906,34 @@ To detect the vulnerbility of Ruby on Rails v4.2.1, cpeNames needs to be set in "cpe:/a:rubyonrails:ruby_on_rails:4.2.1", ] ``` + +# Usage: Integrate with OWASP Dependency Check to Automatic update when the libraries are updated (Experimental) +[OWASP Dependency check](https://www.owasp.org/index.php/OWASP_Dependency_Check) is a utility that identifies project dependencies and checks if there are any known, publicly disclosed, vulnerabilities. + +Benefit of integrating Vuls And OWASP Dependency Check is below. +- Automatic Update of Vuls config when the libraries are updated. +- Reporting by Email or Slack by using Vuls. +- Reporting in Japanese + - OWASP Dependency Check supports only English + +How to integrate Vuls with OWASP Dependency Check +- Execute OWASP Dependency Check with --format=XML option. +- Define the xml file path of dependency check in config.toml. + + ``` + [servers] + + [servers.172-31-4-82] + host = "172.31.4.82" + user = "ec2-user" + keyPath = "/home/username/.ssh/id_rsa" + dependencyCheckXMLPath = "/tmp/dependency-check-report.xml" + ``` + # Usage: Scan Docker containers -It is common that keep Docker containers runnning without SSHd daemon. +It is common that keep Docker containers running without SSHd daemon. see [Docker Blog:Why you don't need to run SSHd in your Docker containers](https://blog.docker.com/2014/06/why-you-dont-need-to-run-sshd-in-docker/) Vuls scans Docker containers via `docker exec` instead of SSH. @@ -901,14 +985,14 @@ tui: ``` -Key binding is bellow. +Key binding is below. | key | | |:-----------------|:-------|:------| | TAB | move cursor among the panes | | Arrow up/down | move cursor to up/down | -| Ctrl+j, Ctrl+k | move cursor to up/donw | -| Ctrl+u, Ctrl+d | page up/donw | +| Ctrl+j, Ctrl+k | move cursor to up/down | +| Ctrl+u, Ctrl+d | page up/down | For details, see https://github.com/future-architect/vuls/blob/master/report/tui.go @@ -954,81 +1038,30 @@ $ vuls scan -cve-dictionary-url=http://192.168.0.1:1323 # Usage: Update NVD Data -``` -$ go-cve-dictionary fetchnvd -h -fetchnvd: - fetchnvd - [-last2y] - [-dbpath=/path/to/cve.sqlite3] - [-debug] - [-debug-sql] - - -dbpath string - /path/to/sqlite3 (default "$PWD/cve.sqlite3") - -debug - debug mode - -debug-sql - SQL debug mode - -last2y - Refresh NVD data in the last two years. -``` - -- Fetch data of the entire period - -``` -$ go-cve-dictionary fetchnvd -entire -``` - -- Fetch data in the last 2 years - -``` -$ go-cve-dictionary fetchnvd -last2y -``` - ----- - -# Deploy With Glide +see [go-cve-dictionary#usage-fetch-nvd-data](https://github.com/kotakanbe/go-cve-dictionary#usage-fetch-nvd-data) -If an error occurred while go get, try deploying with glide. -- Install [Glide](https://github.com/Masterminds/glide) -- Deploy go-cve-dictionary -``` -$ go get -d github.com/kotakanbe/go-cve-dictionary -$ cd $GOPATH/src/github.com/kotakanbe/go-cve-dictionary -$ glide install -$ go install -``` -- Deploy vuls -``` -$ go get -d github.com/future-architect/vuls -$ cd $GOPATH/src/github.com/future-architect/vuls -$ glide install -$ go install -``` -- The binaries are created under $GOPARH/bin ---- # Update Vuls With Glide - Update go-cve-dictionary -If the DB schema was changed, please specify new SQLite3 DB file. +If the DB schema was changed, please specify new SQLite3 or MySQL DB file. ``` $ cd $GOPATH/src/github.com/kotakanbe/go-cve-dictionary $ git pull -$ glide install -$ go install +$ mv vendor /tmp/foo +$ make install ``` - Update vuls ``` $ cd $GOPATH/src/github.com/future-architect/vuls $ git pull -$ glide install -$ go install +$ mv vendor /tmp/bar +$ make install ``` - -Binary Files are created under $GOPARH/bin +Binary file was built under $GOPARH/bin --- @@ -1044,7 +1077,7 @@ If your system is behind HTTP proxy, you have to specify --http-proxy option. - How to Daemonize go-cve-dictionary Use Systemd, Upstart or supervisord, daemontools... -- How to Enable Automatic-Update of Vunerability Data. +- How to Enable Automatic-Update of Vulnerability Data. Use job scheduler like Cron (with -last2y option). - How to Enable Automatic-Scan. @@ -1119,7 +1152,6 @@ Please see [CHANGELOG](https://github.com/future-architect/vuls/blob/master/CHAN ---- -# Licence +# License Please see [LICENSE](https://github.com/future-architect/vuls/blob/master/LICENSE). - diff --git a/commands/configtest.go b/commands/configtest.go index c2413cdb8a..e80f6f4f8a 100644 --- a/commands/configtest.go +++ b/commands/configtest.go @@ -18,6 +18,7 @@ along with this program. If not, see . package commands import ( + "context" "flag" "io/ioutil" "os" @@ -26,7 +27,6 @@ import ( "github.com/Sirupsen/logrus" "github.com/google/subcommands" - "golang.org/x/net/context" c "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/scan" diff --git a/commands/discover.go b/commands/discover.go index 9e66c55ebc..4aa32e2f57 100644 --- a/commands/discover.go +++ b/commands/discover.go @@ -18,6 +18,7 @@ along with this program. If not, see . package commands import ( + "context" "flag" "fmt" "os" @@ -25,7 +26,6 @@ import ( "text/template" "github.com/google/subcommands" - "golang.org/x/net/context" "github.com/Sirupsen/logrus" ps "github.com/kotakanbe/go-pingscanner" @@ -100,7 +100,7 @@ notifyUsers = ["@username"] [mail] smtpAddr = "smtp.gmail.com" -smtpPort = "465" +smtpPort = "587" user = "username" password = "password" from = "from@address.com" @@ -115,7 +115,9 @@ subjectPrefix = "[vuls]" #cpeNames = [ # "cpe:/a:rubyonrails:ruby_on_rails:4.2.1", #] +#dependencyCheckXMLPath = "/tmp/dependency-check-report.xml" #containers = ["${running}"] +#ignoreCves = ["CVE-2014-6271"] #optional = [ # ["key", "value"], #] @@ -131,7 +133,9 @@ host = "{{$ip}}" #cpeNames = [ # "cpe:/a:rubyonrails:ruby_on_rails:4.2.1", #] +#dependencyCheckXMLPath = "/tmp/dependency-check-report.xml" #containers = ["${running}"] +#ignoreCves = ["CVE-2014-0160"] #optional = [ # ["key", "value"], #] diff --git a/commands/history.go b/commands/history.go index 4d5644a81f..9593286650 100644 --- a/commands/history.go +++ b/commands/history.go @@ -18,6 +18,7 @@ along with this program. If not, see . package commands import ( + "context" "flag" "fmt" "io/ioutil" @@ -28,7 +29,6 @@ import ( c "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/report" "github.com/google/subcommands" - "golang.org/x/net/context" ) // HistoryCmd is Subcommand of list scanned results @@ -81,10 +81,6 @@ func (p *HistoryCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{ } var hosts []string for _, f := range files { - // TODO this "if block" will be deleted in a future release - if f.Name() == "all.json" { - continue - } if filepath.Ext(f.Name()) != ".json" { continue } diff --git a/commands/prepare.go b/commands/prepare.go index 5e24719f15..c44bc87653 100644 --- a/commands/prepare.go +++ b/commands/prepare.go @@ -18,6 +18,7 @@ along with this program. If not, see . package commands import ( + "context" "flag" "os" "path/filepath" @@ -27,7 +28,6 @@ import ( "github.com/future-architect/vuls/scan" "github.com/future-architect/vuls/util" "github.com/google/subcommands" - "golang.org/x/net/context" ) // PrepareCmd is Subcommand of host discovery mode @@ -37,6 +37,8 @@ type PrepareCmd struct { askSudoPassword bool askKeyPassword bool + + sshExternal bool } // Name return subcommand name @@ -60,8 +62,9 @@ func (*PrepareCmd) Usage() string { [-config=/path/to/config.toml] [-ask-key-password] [-debug] + [-ssh-external] - [SERVER]... + [SERVER]... ` } @@ -86,8 +89,15 @@ func (p *PrepareCmd) SetFlags(f *flag.FlagSet) { &p.askSudoPassword, "ask-sudo-password", false, - "[Deprecated] THIS OPTION WAS REMOVED FOR SECURITY REASON. Define NOPASSWD in /etc/sudoers on tareget servers and use SSH key-based authentication", + "[Deprecated] THIS OPTION WAS REMOVED FOR SECURITY REASONS. Define NOPASSWD in /etc/sudoers on target servers and use SSH key-based authentication", ) + + f.BoolVar( + &p.sshExternal, + "ssh-external", + false, + "Use external ssh command. Default: Use the Go native implementation") + } // Execute execute @@ -102,7 +112,7 @@ func (p *PrepareCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{ } } if p.askSudoPassword { - logrus.Errorf("[Deprecated] -ask-sudo-password WAS REMOVED FOR SECURITY REASONS. Define NOPASSWD in /etc/sudoers on tareget servers and use SSH key-based authentication") + logrus.Errorf("[Deprecated] -ask-sudo-password WAS REMOVED FOR SECURITY REASONS. Define NOPASSWD in /etc/sudoers on target servers and use SSH key-based authentication") return subcommands.ExitFailure } @@ -133,6 +143,7 @@ func (p *PrepareCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{ } c.Conf.Debug = p.debug + c.Conf.SSHExternal = p.sshExternal // Set up custom logger logger := util.NewCustomLogger(c.ServerInfo{}) @@ -149,10 +160,9 @@ func (p *PrepareCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{ return subcommands.ExitFailure } - logger.Info("Installing...") if errs := scan.Prepare(); 0 < len(errs) { for _, e := range errs { - logger.Errorf("Failed: %s", e) + logger.Errorf("Failed to prepare: %s", e) } return subcommands.ExitFailure } diff --git a/commands/scan.go b/commands/scan.go index 2f7076f32f..896f70fb13 100644 --- a/commands/scan.go +++ b/commands/scan.go @@ -18,6 +18,7 @@ along with this program. If not, see . package commands import ( + "context" "encoding/json" "flag" "fmt" @@ -34,7 +35,7 @@ import ( "github.com/future-architect/vuls/scan" "github.com/future-architect/vuls/util" "github.com/google/subcommands" - "golang.org/x/net/context" + "github.com/k0kubun/pp" ) // ScanCmd is Subcommand of host discovery mode @@ -46,6 +47,7 @@ type ScanCmd struct { configPath string resultsDir string + cvedbtype string cvedbpath string cveDictionaryURL string cacheDBPath string @@ -91,7 +93,8 @@ func (*ScanCmd) Usage() string { [-lang=en|ja] [-config=/path/to/config.toml] [-results-dir=/path/to/results] - [-cve-dictionary-dbpath=/path/to/cve.sqlite3] + [-cve-dictionary-dbtype=sqlite3|mysql] + [-cve-dictionary-dbpath=/path/to/cve.sqlite3 or mysql connection string] [-cve-dictionary-url=http://127.0.0.1:1323] [-cache-dbpath=/path/to/cache.db] [-cvss-over=7] @@ -136,6 +139,12 @@ func (p *ScanCmd) SetFlags(f *flag.FlagSet) { defaultResultsDir := filepath.Join(wd, "results") f.StringVar(&p.resultsDir, "results-dir", defaultResultsDir, "/path/to/results") + f.StringVar( + &p.cvedbtype, + "cve-dictionary-dbtype", + "sqlite3", + "DB type for fetching CVE dictionary (sqlite3 or mysql)") + f.StringVar( &p.cvedbpath, "cve-dictionary-dbpath", @@ -212,7 +221,7 @@ func (p *ScanCmd) SetFlags(f *flag.FlagSet) { f.BoolVar(&p.reportAzureBlob, "report-azure-blob", false, - "Write report to S3 (container/yyyyMMdd_HHmm/servername.json)", + "Write report to Azure Storage blob (container/yyyyMMdd_HHmm/servername.json)", ) f.StringVar(&p.azureAccount, "azure-account", "", "Azure account name to use. AZURE_STORAGE_ACCOUNT environment variable is used if not specified") f.StringVar(&p.azureKey, "azure-key", "", "Azure account key to use. AZURE_STORAGE_ACCESS_KEY environment variable is used if not specified") @@ -229,7 +238,7 @@ func (p *ScanCmd) SetFlags(f *flag.FlagSet) { &p.askSudoPassword, "ask-sudo-password", false, - "[Deprecated] THIS OPTION WAS REMOVED FOR SECURITY REASONS. Define NOPASSWD in /etc/sudoers on tareget servers and use SSH key-based authentication", + "[Deprecated] THIS OPTION WAS REMOVED FOR SECURITY REASONS. Define NOPASSWD in /etc/sudoers on target servers and use SSH key-based authentication", ) } @@ -246,11 +255,12 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) } } if p.askSudoPassword { - logrus.Errorf("[Deprecated] -ask-sudo-password WAS REMOVED FOR SECURITY REASONS. Define NOPASSWD in /etc/sudoers on tareget servers and use SSH key-based authentication") + logrus.Errorf("[Deprecated] -ask-sudo-password WAS REMOVED FOR SECURITY REASONS. Define NOPASSWD in /etc/sudoers on target servers and use SSH key-based authentication") return subcommands.ExitFailure } // Load up the config file here + c.Conf.Debug = p.debug err = c.Load(p.configPath, keyPass) if err != nil { logrus.Errorf("Error loading %s, %s", p.configPath, err) @@ -260,7 +270,9 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) logrus.Info("Start scanning") logrus.Infof("config: %s", p.configPath) if p.cvedbpath != "" { - logrus.Infof("cve-dictionary: %s", p.cvedbpath) + if p.cvedbtype == "sqlite3" { + logrus.Infof("cve-dictionary: %s", p.cvedbpath) + } } else { logrus.Infof("cve-dictionary: %s", p.cveDictionaryURL) } @@ -301,9 +313,9 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) if 0 < len(servernames) { c.Conf.Servers = target } + logrus.Debugf("%s", pp.Sprintf("%v", target)) c.Conf.Lang = p.lang - c.Conf.Debug = p.debug c.Conf.DebugSQL = p.debugSQL // logger @@ -363,6 +375,7 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) } c.Conf.ResultsDir = p.resultsDir + c.Conf.CveDBType = p.cvedbtype c.Conf.CveDBPath = p.cvedbpath c.Conf.CveDictionaryURL = p.cveDictionaryURL c.Conf.CacheDBPath = p.cacheDBPath diff --git a/commands/tui.go b/commands/tui.go index 4c7c150240..1b9cd53fc7 100644 --- a/commands/tui.go +++ b/commands/tui.go @@ -18,6 +18,7 @@ along with this program. If not, see . package commands import ( + "context" "flag" "io/ioutil" "os" @@ -28,7 +29,6 @@ import ( c "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/report" "github.com/google/subcommands" - "golang.org/x/net/context" ) // TuiCmd is Subcommand of host discovery mode diff --git a/config/config.go b/config/config.go index 919ad21574..3002adda59 100644 --- a/config/config.go +++ b/config/config.go @@ -51,6 +51,7 @@ type Config struct { HTTPProxy string `valid:"url"` ResultsDir string + CveDBType string CveDBPath string CacheDBPath string @@ -77,10 +78,23 @@ func (c Config) Validate() bool { } } - if len(c.CveDBPath) != 0 { - if ok, _ := valid.IsFilePath(c.CveDBPath); !ok { - errs = append(errs, fmt.Errorf( - "SQLite3 DB(Cve Dictionary) path must be a *Absolute* file path. -cve-dictionary-dbpath: %s", c.CveDBPath)) + // If no valid DB type is set, default to sqlite3 + if c.CveDBType == "" { + c.CveDBType = "sqlite3" + } + + if c.CveDBType != "sqlite3" && c.CveDBType != "mysql" { + errs = append(errs, fmt.Errorf( + "CVE DB type must be either 'sqlite3' or 'mysql'. -cve-dictionary-dbtype: %s", c.CveDBType)) + } + + + if c.CveDBType == "sqlite3" { + if len(c.CveDBPath) != 0 { + if ok, _ := valid.IsFilePath(c.CveDBPath); !ok { + errs = append(errs, fmt.Errorf( + "SQLite3 DB(Cve Dictionary) path must be a *Absolute* file path. -cve-dictionary-dbpath: %s", c.CveDBPath)) + } } } @@ -190,7 +204,6 @@ type SlackConf struct { // Validate validates configuration func (c *SlackConf) Validate() (errs []error) { - if !c.UseThisTime { return } @@ -230,11 +243,14 @@ type ServerInfo struct { KeyPath string KeyPassword string - CpeNames []string + CpeNames []string + DependencyCheckXMLPath string // Container Names or IDs Containers []string + IgnoreCves []string + // Optional key-value set that will be outputted to JSON Optional [][]interface{} diff --git a/config/tomlloader.go b/config/tomlloader.go index c0fbb0fb19..9bad2390be 100644 --- a/config/tomlloader.go +++ b/config/tomlloader.go @@ -23,7 +23,7 @@ import ( "github.com/BurntSushi/toml" log "github.com/Sirupsen/logrus" - "github.com/k0kubun/pp" + "github.com/future-architect/vuls/contrib/owasp-dependency-check/parser" ) // TOMLLoader loads config @@ -31,7 +31,11 @@ type TOMLLoader struct { } // Load load the configuraiton TOML file specified by path arg. -func (c TOMLLoader) Load(pathToToml, keyPass string) (err error) { +func (c TOMLLoader) Load(pathToToml, keyPass string) error { + if Conf.Debug { + log.SetLevel(log.DebugLevel) + } + var conf Config if _, err := toml.DecodeFile(pathToToml, &conf); err != nil { log.Error("Load config failed", err) @@ -52,7 +56,6 @@ func (c TOMLLoader) Load(pathToToml, keyPass string) (err error) { } i := 0 for name, v := range conf.Servers { - if 0 < len(v.KeyPassword) { log.Warn("[Deprecated] KEYPASSWORD IN CONFIG FILE ARE UNSECURE. REMOVE THEM IMMEDIATELY FOR A SECURITY REASONS. THEY WILL BE REMOVED IN A FUTURE RELEASE.") } @@ -104,11 +107,42 @@ func (c TOMLLoader) Load(pathToToml, keyPass string) (err error) { s.CpeNames = d.CpeNames } + s.DependencyCheckXMLPath = v.DependencyCheckXMLPath + if len(s.DependencyCheckXMLPath) == 0 { + s.DependencyCheckXMLPath = d.DependencyCheckXMLPath + } + + // Load CPEs from OWASP Dependency Check XML + if len(s.DependencyCheckXMLPath) != 0 { + cpes, err := parser.Parse(s.DependencyCheckXMLPath) + if err != nil { + return fmt.Errorf( + "Failed to read OWASP Dependency Check XML: %s", err) + } + log.Infof("Loaded from OWASP Dependency Check XML: %s", + s.ServerName) + s.CpeNames = append(s.CpeNames, cpes...) + } + s.Containers = v.Containers if len(s.Containers) == 0 { s.Containers = d.Containers } + s.IgnoreCves = v.IgnoreCves + for _, cve := range d.IgnoreCves { + found := false + for _, c := range s.IgnoreCves { + if cve == c { + found = true + break + } + } + if !found { + s.IgnoreCves = append(s.IgnoreCves, cve) + } + } + s.Optional = v.Optional for _, dkv := range d.Optional { found := false @@ -128,8 +162,6 @@ func (c TOMLLoader) Load(pathToToml, keyPass string) (err error) { servers[name] = s } - log.Debug("Config loaded") - log.Debugf("%s", pp.Sprintf("%v", servers)) Conf.Servers = servers - return + return nil } diff --git a/contrib/owasp-dependency-check/parser/parser.go b/contrib/owasp-dependency-check/parser/parser.go new file mode 100644 index 0000000000..c0e8e81884 --- /dev/null +++ b/contrib/owasp-dependency-check/parser/parser.go @@ -0,0 +1,64 @@ +package parser + +import ( + "encoding/xml" + "fmt" + "io/ioutil" + "os" + "sort" + "strings" +) + +type analysis struct { + Dependencies []dependency `xml:"dependencies>dependency"` +} + +type dependency struct { + Identifiers []identifier `xml:"identifiers>identifier"` +} + +type identifier struct { + Name string `xml:"name"` + Type string `xml:"type,attr"` +} + +func appendIfMissing(slice []string, str string) []string { + for _, s := range slice { + if s == str { + return slice + } + } + return append(slice, str) +} + +// Parse parses XML and collect list of cpe +func Parse(path string) ([]string, error) { + file, err := os.Open(path) + if err != nil { + return []string{}, fmt.Errorf("Failed to open: %s", err) + } + defer file.Close() + + b, err := ioutil.ReadAll(file) + if err != nil { + return []string{}, fmt.Errorf("Failed to read: %s", err) + } + + var anal analysis + if err := xml.Unmarshal(b, &anal); err != nil { + fmt.Errorf("Failed to unmarshal: %s", err) + } + + cpes := []string{} + for _, d := range anal.Dependencies { + for _, ident := range d.Identifiers { + if ident.Type == "cpe" { + name := strings.TrimPrefix(ident.Name, "(") + name = strings.TrimSuffix(name, ")") + cpes = appendIfMissing(cpes, name) + } + } + } + sort.Strings(cpes) + return cpes, nil +} diff --git a/cveapi/cve_client.go b/cveapi/cve_client.go index eb33838183..44f5f282b6 100644 --- a/cveapi/cve_client.go +++ b/cveapi/cve_client.go @@ -49,7 +49,7 @@ func (api *cvedictClient) initialize() { func (api cvedictClient) CheckHealth() (ok bool, err error) { if config.Conf.CveDBPath != "" { - log.Debugf("get cve-dictionary from sqlite3") + log.Debugf("get cve-dictionary from %s", config.Conf.CveDBType) return true, nil } @@ -135,8 +135,10 @@ func (api cvedictClient) FetchCveDetails(cveIDs []string) (cveDetails cve.CveDet } func (api cvedictClient) FetchCveDetailsFromCveDB(cveIDs []string) (cveDetails cve.CveDetails, err error) { - log.Debugf("open cve-dictionary db") + log.Debugf("open cve-dictionary db (%s)", config.Conf.CveDBType) + cveconfig.Conf.DBType = config.Conf.CveDBType cveconfig.Conf.DBPath = config.Conf.CveDBPath + cveconfig.Conf.DebugSQL = config.Conf.DebugSQL if err := cvedb.OpenDB(); err != nil { return []cve.CveDetail{}, fmt.Errorf("Failed to open DB. err: %s", err) @@ -239,8 +241,11 @@ func (api cvedictClient) httpPost(key, url string, query map[string]string) ([]c } func (api cvedictClient) FetchCveDetailsByCpeNameFromDB(cpeName string) ([]cve.CveDetail, error) { - log.Debugf("open cve-dictionary db") + log.Debugf("open cve-dictionary db (%s)", config.Conf.CveDBType) + cveconfig.Conf.DBType = config.Conf.CveDBType cveconfig.Conf.DBPath = config.Conf.CveDBPath + cveconfig.Conf.DebugSQL = config.Conf.DebugSQL + if err := cvedb.OpenDB(); err != nil { return []cve.CveDetail{}, fmt.Errorf("Failed to open DB. err: %s", err) diff --git a/glide.lock b/glide.lock index efb4c1e8d6..bdc53d2bdc 100644 --- a/glide.lock +++ b/glide.lock @@ -1,10 +1,10 @@ -hash: 28d14f88e90c0765c1b660ddde796e51e197239d353bb79bfc5d8f8cf9b5f9ee -updated: 2016-09-22T12:01:23.554218373-07:00 +hash: ca64aef6e9e94c7be91f79b88edb847363c8a5bd48da4ad27784e9342c8db6e2 +updated: 2016-11-02T12:36:18.306508648-07:00 imports: - name: github.com/asaskevich/govalidator - version: 593d64559f7600f29581a3ee42177f5dbded27a9 + version: 7b3beb6df3c42abd3509abfc3bcacc0fbfb7c877 - name: github.com/aws/aws-sdk-go - version: 54888dfe736c54ceb65ccd5443d17928d90034c4 + version: 967a4b67114c61521c4b514d7c9a2feb869918e9 subpackages: - aws - aws/credentials @@ -32,44 +32,48 @@ imports: - private/protocol/xml/xmlutil - private/protocol/query/queryutil - name: github.com/Azure/azure-sdk-for-go - version: 63d3f3e3b12ffb726ba3f72fef1aa8c1c7cd1012 + version: bd73d950fa4440dae889bd9917bff7cef539f86e subpackages: - storage - name: github.com/boltdb/bolt - version: fff57c100f4dea1905678da7e90d92429dff2904 + version: 4b1ebc1869ad66568b313d0dc410e2be72670dda - name: github.com/BurntSushi/toml version: 99064174e013895bbd9b025c31100bd1d9b590ca - name: github.com/cenkalti/backoff - version: 8edc80b07f38c27352fb186d971c628a6c32552b + version: b02f2bbce11d7ea6b97f282ef1771b0fe2f65ef3 - name: github.com/cheggaaa/pb - version: ad4efe000aa550bb54918c06ebbadc0ff17687b9 + version: dd61faab99a777c652bb680e37715fe0cb549856 - name: github.com/go-ini/ini version: 6e4869b434bd001f6983749881c7ead3545887d8 +- name: github.com/go-sql-driver/mysql + version: ce924a41eea897745442daaa1739089b0f3f561d - name: github.com/google/subcommands - version: 1c7173745a6001f67d8d96ab4e178284c77f7759 + version: a71b91e238406bd68766ee52db63bebedce0e9f6 - name: github.com/gosuri/uitable version: 36ee7e946282a3fb1cfecd476ddc9b35d8847e42 subpackages: - util/strutil - util/wordwrap - name: github.com/howeyc/gopass - version: 26c6e1184fd5255fa5f5289d0b789a4819c203a4 + version: f5387c492211eb133053880d23dfae62aa14123d - name: github.com/jinzhu/gorm - version: 041cd3dd31a68a3e9ad20fcddb46085df5aebc76 + version: 4a540f3ac83a6765b8876be67994ecde022c6198 + subpackages: + - dialects/mysql - name: github.com/jinzhu/inflection version: 74387dc39a75e970e7a3ae6a3386b5bd2e5c5cff - name: github.com/jmespath/go-jmespath version: bd40a432e4c76585ef6b72d3fd96fb9b6dc7b68d - name: github.com/jroimartin/gocui - version: 30f7d65597dc2c421ce452b164c36b7014ef94be + version: 0975ddb2a85e94dfb9fcd0b8672f4a33b960fb1f - name: github.com/k0kubun/pp version: f5dce6ed0ccf6c350f1679964ff6b61f3d6d2033 - name: github.com/kotakanbe/go-cve-dictionary - version: f9f68fee57dca8e60fb5d9d6b34d3215d854fc06 + version: 70989b6709c3102924ad8c8483e9bdc99bcb598b subpackages: - config - - models - db + - models - log - jvn - nvd @@ -79,27 +83,27 @@ imports: - name: github.com/kotakanbe/logrus-prefixed-formatter version: f4f7d41649cf1e75e736884da8d05324aa76ea25 - name: github.com/mattn/go-colorable - version: ed8eb9e318d7a84ce5915b495b7d35e0cfe7b5a8 + version: 6e26b354bd2b0fc420cb632b0d878abccdc6544c - name: github.com/mattn/go-isatty version: 66b8e73f3f5cda9f96b69efd03dd3d7fc4a5cdb8 - name: github.com/mattn/go-runewidth - version: d6bea18f789704b5f83375793155289da36a3c7f + version: 737072b4e32b7a5018b4a7125da8d12de90e8045 - name: github.com/mattn/go-sqlite3 - version: 3fb7a0e792edd47bf0cf1e919dfc14e2be412e15 + version: 86681de00adef4f8040947b7d35f97000fc5a230 - name: github.com/mgutz/ansi version: c286dcecd19ff979eeb73ea444e479b903f2cfcb - name: github.com/moul/http2curl - version: b1479103caacaa39319f75e7f57fc545287fca0d + version: 4e24498b31dba4683efb9d35c1c8a91e2eda28c8 - name: github.com/nsf/termbox-go version: b6acae516ace002cb8105a89024544a1480655a5 - name: github.com/parnurzeal/gorequest - version: 29ced6f360a5ac3823a3675b4e29fbab336cc186 + version: b8943263f1ae87258d26a5c2181d1bc71deeddc0 - name: github.com/rifflock/lfshook version: 3f9d976bd7402de39b46357069fb6325a974572e - name: github.com/Sirupsen/logrus - version: 3ec0642a7fb6488f65b06f9040adc67e3990296a + version: 380f64d344b252a007a59baa61f31820f59cba89 - name: golang.org/x/crypto - version: 8e06e8ddd9629eb88639aba897641bff8031f1d3 + version: 9477e0b78b9ac3d0b03822fd95422e2fe07627cd subpackages: - ssh - ssh/agent @@ -108,16 +112,12 @@ imports: - ed25519 - ed25519/internal/edwards25519 - name: golang.org/x/net - version: 6d3beaea10370160dea67f5c9327ed791afd5389 + version: 4bb47a1098b37d69980d96237e2ae4ff56bb5a95 subpackages: - context - publicsuffix - name: golang.org/x/sys - version: 8f0908ab3b2457e2e15403d3697c9ef5cb4b57a9 + version: c200b10b5d5e122be351b67af224adc6128af5bf subpackages: - unix -- name: gopkg.in/alexcesaro/quotedprintable.v3 - version: 2caba252f4dc53eaf6b553000885530023f54623 -- name: gopkg.in/gomail.v2 - version: 81ebce5c23dfd25c6c67194b37d3dd3f338c98b1 testImports: [] diff --git a/glide.yaml b/glide.yaml index 015240a5dd..4608cf17aa 100644 --- a/glide.yaml +++ b/glide.yaml @@ -12,6 +12,7 @@ import: - aws/credentials - aws/session - service/s3 +- package: github.com/boltdb/bolt - package: github.com/cenkalti/backoff - package: github.com/google/subcommands - package: github.com/gosuri/uitable @@ -22,6 +23,7 @@ import: - package: github.com/kotakanbe/go-cve-dictionary subpackages: - config + - db - models - package: github.com/kotakanbe/go-pingscanner - package: github.com/kotakanbe/logrus-prefixed-formatter @@ -35,4 +37,3 @@ import: - package: golang.org/x/net subpackages: - context -- package: gopkg.in/gomail.v2 diff --git a/main.go b/main.go index 35acb99b0e..e96c6bce49 100644 --- a/main.go +++ b/main.go @@ -22,15 +22,20 @@ import ( "fmt" "os" - "golang.org/x/net/context" + "context" "github.com/future-architect/vuls/commands" - "github.com/future-architect/vuls/version" "github.com/google/subcommands" _ "github.com/mattn/go-sqlite3" ) +// Version of Vuls +var version = "0.1.6" + +// Revision of Git +var revision string + func main() { subcommands.Register(subcommands.HelpCommand(), "") subcommands.Register(subcommands.FlagsCommand(), "") @@ -47,7 +52,7 @@ func main() { flag.Parse() if *v { - fmt.Printf("%s %s\n", version.Name, version.Version) + fmt.Printf("vuls %s %s\n", version, revision) os.Exit(int(subcommands.ExitSuccess)) } diff --git a/models/models.go b/models/models.go index 7c862cfe18..3a1877748d 100644 --- a/models/models.go +++ b/models/models.go @@ -89,6 +89,7 @@ type ScanResult struct { // NWLinks []NWLink KnownCves []CveInfo UnknownCves []CveInfo + IgnoredCves []CveInfo Optional [][]interface{} `gorm:"-"` } diff --git a/report/json.go b/report/json.go index 4781187453..acff8c8921 100644 --- a/report/json.go +++ b/report/json.go @@ -61,14 +61,6 @@ func (w JSONWriter) Write(scanResults []models.ScanResult) (err error) { } var jsonBytes []byte - if jsonBytes, err = json.Marshal(scanResults); err != nil { - return fmt.Errorf("Failed to Marshal to JSON: %s", err) - } - all := filepath.Join(path, "all.json") - if err := ioutil.WriteFile(all, jsonBytes, 0600); err != nil { - return fmt.Errorf("Failed to write JSON. path: %s, err: %s", all, err) - } - for _, r := range scanResults { jsonPath := "" if len(r.Container.ContainerID) == 0 { @@ -117,10 +109,6 @@ func LoadOneScanHistory(jsonDir string) (scanHistory models.ScanHistory, err err return } for _, file := range files { - // TODO this "if block" will be deleted in a future release - if file.Name() == "all.json" { - continue - } if filepath.Ext(file.Name()) != ".json" { continue } diff --git a/report/mail.go b/report/mail.go index 34cd3febde..21803874be 100644 --- a/report/mail.go +++ b/report/mail.go @@ -18,13 +18,14 @@ along with this program. If not, see . package report import ( - "crypto/tls" "fmt" - "strconv" + "net" + "net/mail" + "net/smtp" + "strings" "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/models" - "gopkg.in/gomail.v2" ) // MailWriter send mail @@ -33,37 +34,53 @@ type MailWriter struct{} func (w MailWriter) Write(scanResults []models.ScanResult) (err error) { conf := config.Conf for _, s := range scanResults { - m := gomail.NewMessage() - m.SetHeader("From", conf.Mail.From) - m.SetHeader("To", conf.Mail.To...) - m.SetHeader("Cc", conf.Mail.Cc...) + to := strings.Join(conf.Mail.To[:], ", ") + cc := strings.Join(conf.Mail.Cc[:], ", ") + mailAddresses := append(conf.Mail.To, conf.Mail.Cc...) + if _, err := mail.ParseAddressList(strings.Join(mailAddresses[:], ", ")); err != nil { + return fmt.Errorf("Failed to parse email addresses: %s", err) + } subject := fmt.Sprintf("%s%s %s", conf.Mail.SubjectPrefix, s.ServerInfo(), s.CveSummary(), ) - m.SetHeader("Subject", subject) + + headers := make(map[string]string) + headers["From"] = conf.Mail.From + headers["To"] = to + headers["Cc"] = cc + headers["Subject"] = subject + + var message string + for k, v := range headers { + message += fmt.Sprintf("%s: %s\r\n", k, v) + } var body string if body, err = toPlainText(s); err != nil { return err } - m.SetBody("text/plain", body) - port, _ := strconv.Atoi(conf.Mail.SMTPPort) - d := gomail.NewPlainDialer( - conf.Mail.SMTPAddr, - port, - conf.Mail.User, - conf.Mail.Password, - ) + message += "\r\n" + body - d.TLSConfig = &tls.Config{ - InsecureSkipVerify: true, - } + smtpServer := net.JoinHostPort(conf.Mail.SMTPAddr, conf.Mail.SMTPPort) + + err := smtp.SendMail( + smtpServer, + smtp.PlainAuth( + "", + conf.Mail.User, + conf.Mail.Password, + conf.Mail.SMTPAddr, + ), + conf.Mail.From, + conf.Mail.To, + []byte(message), + ) - if err := d.DialAndSend(m); err != nil { - panic(err) + if err != nil { + return fmt.Errorf("Failed to send emails: %s", err) } } return nil diff --git a/report/s3.go b/report/s3.go index 95b7951ea8..f55484101a 100644 --- a/report/s3.go +++ b/report/s3.go @@ -78,16 +78,6 @@ func (w S3Writer) Write(scanResults []models.ScanResult) (err error) { // http://docs.aws.amazon.com/sdk-for-go/latest/v1/developerguide/common-examples.title.html svc := getS3() timestr := time.Now().Format("20060102_1504") - key := fmt.Sprintf("%s/%s", timestr, "all.json") - _, err = svc.PutObject(&s3.PutObjectInput{ - Bucket: &c.Conf.S3Bucket, - Key: &key, - Body: bytes.NewReader(jsonBytes), - }) - if err != nil { - return fmt.Errorf("Failed to upload data to %s/%s, %s", c.Conf.S3Bucket, key, err) - } - for _, r := range scanResults { key := "" if len(r.Container.ContainerID) == 0 { diff --git a/report/slack.go b/report/slack.go index c0faa437a1..655445e67f 100644 --- a/report/slack.go +++ b/report/slack.go @@ -59,7 +59,6 @@ type SlackWriter struct{} func (w SlackWriter) Write(scanResults []models.ScanResult) error { conf := config.Conf.Slack for _, s := range scanResults { - channel := conf.Channel if channel == "${servername}" { channel = fmt.Sprintf("#%s", s.ServerName) @@ -97,7 +96,6 @@ func (w SlackWriter) Write(scanResults []models.ScanResult) error { } func msgText(r models.ScanResult) string { - notifyUsers := "" if 0 < len(r.KnownCves) || 0 < len(r.UnknownCves) { notifyUsers = getNotifyUsers(config.Conf.Slack.NotifyUsers) @@ -108,7 +106,6 @@ func msgText(r models.ScanResult) string { } func toSlackAttachments(scanResult models.ScanResult) (attaches []*attachment) { - cves := scanResult.KnownCves if !config.Conf.IgnoreUnscoredCves { cves = append(cves, scanResult.UnknownCves...) diff --git a/report/tui.go b/report/tui.go index 4a41635bc8..2b2477d2fc 100644 --- a/report/tui.go +++ b/report/tui.go @@ -44,26 +44,28 @@ func RunTui(jsonDirName string) subcommands.ExitStatus { var err error scanHistory, err = selectScanHistory(jsonDirName) if err != nil { - log.Fatal(err) + log.Errorf("%s", err) return subcommands.ExitFailure } - g := gocui.NewGui() - if err := g.Init(); err != nil { - log.Panicln(err) + g, err := gocui.NewGui() + if err != nil { + log.Errorf("%s", err) + return subcommands.ExitFailure } defer g.Close() - g.SetLayout(layout) + g.SetManagerFunc(layout) if err := keybindings(g); err != nil { - log.Panicln(err) + log.Errorf("%s", err) + return subcommands.ExitFailure } g.SelBgColor = gocui.ColorGreen g.SelFgColor = gocui.ColorBlack g.Cursor = true if err := g.MainLoop(); err != nil && err != gocui.ErrQuit { - log.Panicln(err) + log.Errorf("%s", err) return subcommands.ExitFailure } @@ -174,35 +176,41 @@ func keybindings(g *gocui.Gui) (err error) { } func nextView(g *gocui.Gui, v *gocui.View) error { + var err error + if v == nil { - return g.SetCurrentView("side") + _, err = g.SetCurrentView("side") } switch v.Name() { case "side": - return g.SetCurrentView("summary") + _, err = g.SetCurrentView("summary") case "summary": - return g.SetCurrentView("detail") + _, err = g.SetCurrentView("detail") case "detail": - return g.SetCurrentView("side") + _, err = g.SetCurrentView("side") default: - return g.SetCurrentView("summary") + _, err = g.SetCurrentView("summary") } + return err } func previousView(g *gocui.Gui, v *gocui.View) error { + var err error + if v == nil { - return g.SetCurrentView("side") + _, err = g.SetCurrentView("side") } switch v.Name() { case "side": - return g.SetCurrentView("side") + _, err = g.SetCurrentView("side") case "summary": - return g.SetCurrentView("side") + _, err = g.SetCurrentView("side") case "detail": - return g.SetCurrentView("summary") + _, err = g.SetCurrentView("summary") default: - return g.SetCurrentView("side") + _, err = g.SetCurrentView("side") } + return err } func movable(v *gocui.View, nextY int) (ok bool, yLimit int) { @@ -371,7 +379,7 @@ func cursorPageUp(g *gocui.Gui, v *gocui.View) error { func previousSummary(g *gocui.Gui, v *gocui.View) error { if v != nil { // cursor to summary - if err := g.SetCurrentView("summary"); err != nil { + if _, err := g.SetCurrentView("summary"); err != nil { return err } // move next line @@ -379,7 +387,7 @@ func previousSummary(g *gocui.Gui, v *gocui.View) error { return err } // cursor to detail - if err := g.SetCurrentView("detail"); err != nil { + if _, err := g.SetCurrentView("detail"); err != nil { return err } } @@ -389,7 +397,7 @@ func previousSummary(g *gocui.Gui, v *gocui.View) error { func nextSummary(g *gocui.Gui, v *gocui.View) error { if v != nil { // cursor to summary - if err := g.SetCurrentView("summary"); err != nil { + if _, err := g.SetCurrentView("summary"); err != nil { return err } // move next line @@ -397,7 +405,7 @@ func nextSummary(g *gocui.Gui, v *gocui.View) error { return err } // cursor to detail - if err := g.SetCurrentView("detail"); err != nil { + if _, err := g.SetCurrentView("detail"); err != nil { return err } } @@ -462,7 +470,7 @@ func getLine(g *gocui.Gui, v *gocui.View) error { return err } fmt.Fprintln(v, l) - if err := g.SetCurrentView("msg"); err != nil { + if _, err := g.SetCurrentView("msg"); err != nil { return err } } @@ -484,7 +492,7 @@ func showMsg(g *gocui.Gui, v *gocui.View) error { return err } fmt.Fprintln(v, l) - if err := g.SetCurrentView("msg"); err != nil { + if _, err := g.SetCurrentView("msg"); err != nil { return err } } @@ -495,7 +503,7 @@ func delMsg(g *gocui.Gui, v *gocui.View) error { if err := g.DeleteView("msg"); err != nil { return err } - if err := g.SetCurrentView("summary"); err != nil { + if _, err := g.SetCurrentView("summary"); err != nil { return err } return nil @@ -530,7 +538,7 @@ func setSideLayout(g *gocui.Gui) error { fmt.Fprintln(v, result.ServerInfoTui()) } currentScanResult = scanHistory.ScanResults[0] - if err := g.SetCurrentView("side"); err != nil { + if _, err := g.SetCurrentView("side"); err != nil { return err } } diff --git a/scan/base.go b/scan/base.go index 7b9c3c81e5..a7361575a8 100644 --- a/scan/base.go +++ b/scan/base.go @@ -33,8 +33,9 @@ import ( type base struct { ServerInfo config.ServerInfo Distro config.Distro + Platform models.Platform - Platform models.Platform + lackDependencies []string osPackages log *logrus.Entry @@ -85,6 +86,10 @@ func (l base) getPlatform() models.Platform { return l.Platform } +func (l base) getLackDependencies() []string { + return l.lackDependencies +} + func (l base) allContainers() (containers []config.Container, err error) { switch l.ServerInfo.Container.Type { case "", "docker": @@ -227,13 +232,31 @@ func (l base) isAwsInstanceID(str string) bool { } func (l *base) convertToModel() (models.ScanResult, error) { - var scoredCves, unscoredCves models.CveInfos + var scoredCves, unscoredCves, ignoredCves models.CveInfos for _, p := range l.UnsecurePackages { + // ignoreCves + found := false + for _, icve := range l.getServerInfo().IgnoreCves { + if icve == p.CveDetail.CveID { + ignoredCves = append(ignoredCves, models.CveInfo{ + CveDetail: p.CveDetail, + Packages: p.Packs, + DistroAdvisories: p.DistroAdvisories, + }) + found = true + break + } + } + if found { + continue + } + + // unscoredCves if p.CveDetail.CvssScore(config.Conf.Lang) <= 0 { unscoredCves = append(unscoredCves, models.CveInfo{ CveDetail: p.CveDetail, Packages: p.Packs, - DistroAdvisories: p.DistroAdvisories, // only Amazon Linux + DistroAdvisories: p.DistroAdvisories, }) continue } @@ -244,10 +267,11 @@ func (l *base) convertToModel() (models.ScanResult, error) { models.CpeName{Name: cpename}) } + // scoredCves cve := models.CveInfo{ CveDetail: p.CveDetail, Packages: p.Packs, - DistroAdvisories: p.DistroAdvisories, // only Amazon Linux + DistroAdvisories: p.DistroAdvisories, CpeNames: cpenames, } scoredCves = append(scoredCves, cve) @@ -260,6 +284,7 @@ func (l *base) convertToModel() (models.ScanResult, error) { sort.Sort(scoredCves) sort.Sort(unscoredCves) + sort.Sort(ignoredCves) return models.ScanResult{ ServerName: l.ServerInfo.ServerName, @@ -270,6 +295,7 @@ func (l *base) convertToModel() (models.ScanResult, error) { Platform: l.Platform, KnownCves: scoredCves, UnknownCves: unscoredCves, + IgnoredCves: ignoredCves, Optional: l.ServerInfo.Optional, }, nil } diff --git a/scan/debian.go b/scan/debian.go index cc941c4b21..2f5e757cd5 100644 --- a/scan/debian.go +++ b/scan/debian.go @@ -124,7 +124,31 @@ func (o *debian) checkIfSudoNoPasswd() error { return nil } +func (o *debian) checkDependencies() error { + switch o.Distro.Family { + case "ubuntu": + return nil + + case "debian": + // Debian needs aptitude to get changelogs. + // Because unable to get changelogs via apt-get changelog on Debian. + name := "aptitude" + cmd := name + " -h" + if r := o.ssh(cmd, noSudo); !r.isSuccess() { + o.lackDependencies = []string{name} + } + return nil + + default: + return fmt.Errorf("Not implemented yet: %s", o.Distro) + } +} + func (o *debian) install() error { + if len(o.lackDependencies) == 0 { + return nil + } + // apt-get update o.log.Infof("apt-get update...") cmd := util.PrependProxyEnv("apt-get update") @@ -134,15 +158,14 @@ func (o *debian) install() error { return fmt.Errorf(msg) } - if o.Distro.Family == "debian" { - // install aptitude - cmd = util.PrependProxyEnv("apt-get install --force-yes -y aptitude") + for _, name := range o.lackDependencies { + cmd = util.PrependProxyEnv("apt-get install " + name) if r := o.exec(cmd, sudo); !r.isSuccess() { msg := fmt.Sprintf("Failed to SSH: %s", r) o.log.Errorf(msg) return fmt.Errorf(msg) } - o.log.Infof("Installed: aptitude") + o.log.Infof("Installed: " + name) } return nil } @@ -302,7 +325,7 @@ func (o *debian) fillCandidateVersion(before models.PackageInfoList) (filled []m for _, p := range before { names = append(names, p.Name) } - cmd := fmt.Sprintf("LANG=en_US.UTF-8 apt-cache policy %s", strings.Join(names, " ")) + cmd := fmt.Sprintf("LANGUAGE=en_US.UTF-8 apt-cache policy %s", strings.Join(names, " ")) r := o.exec(cmd, noSudo) if !r.isSuccess() { return nil, fmt.Errorf("Failed to SSH: %s.", r) @@ -324,7 +347,7 @@ func (o *debian) fillCandidateVersion(before models.PackageInfoList) (filled []m } func (o *debian) GetUpgradablePackNames() (packNames []string, err error) { - cmd := util.PrependProxyEnv("LANG=en_US.UTF-8 apt-get upgrade --dry-run") + cmd := util.PrependProxyEnv("LANGUAGE=en_US.UTF-8 apt-get upgrade --dry-run") r := o.exec(cmd, noSudo) if r.isSuccess(0, 1) { return o.parseAptGetUpgrade(r.Stdout) @@ -494,7 +517,7 @@ func (o *debian) getChangelogCache(meta cache.Meta, pack models.PackageInfo) str } changelog, err := cache.DB.GetChangelog(meta.Name, pack.Name) if err != nil { - o.log.Warnf("Failed to get chnagelog. bucket: %s, key:%s, err: %s", + o.log.Warnf("Failed to get changelog. bucket: %s, key:%s, err: %s", meta.Name, pack.Name, err) return "" } @@ -522,11 +545,13 @@ func (o *debian) scanPackageCveIDs(pack models.PackageInfo) ([]string, error) { o.log.Warnf("Failed to SSH: %s", r) // Ignore this Error. return nil, nil - } - err := cache.DB.PutChangelog(o.getServerInfo().GetServerName(), pack.Name, r.Stdout) - if err != nil { - return nil, fmt.Errorf("Failed to put changelog into cache") + + if 0 < len(strings.TrimSpace(r.Stdout)) { + err := cache.DB.PutChangelog(o.getServerInfo().GetServerName(), pack.Name, r.Stdout) + if err != nil { + return nil, fmt.Errorf("Failed to put changelog into cache") + } } // No error will be returned. Only logging. return o.getCveIDFromChangelog(r.Stdout, pack.Name, pack.Version), nil diff --git a/scan/freebsd.go b/scan/freebsd.go index 7bce026fe5..744eb81a19 100644 --- a/scan/freebsd.go +++ b/scan/freebsd.go @@ -65,6 +65,10 @@ func (o *bsd) checkIfSudoNoPasswd() error { return nil } +func (o *bsd) checkDependencies() error { + return nil +} + func (o *bsd) install() error { return nil } diff --git a/scan/freebsd_test.go b/scan/freebsd_test.go index 2351f2b299..48dd62b1a2 100644 --- a/scan/freebsd_test.go +++ b/scan/freebsd_test.go @@ -139,17 +139,17 @@ WWW: https://vuxml.FreeBSD.org/freebsd/ab3e98d9-8175-11e4-907d-d050992ecde8.html d := newBsd(config.ServerInfo{}) for _, tt := range tests { - aName, aCveIDs, aVunlnID := d.parseBlock(tt.in) + aName, aCveIDs, aVulnID := d.parseBlock(tt.in) if tt.name != aName { - t.Errorf("expected vulnID: %s, actual %s", tt.vulnID, aVunlnID) + t.Errorf("expected vulnID: %s, actual %s", tt.vulnID, aVulnID) } for i := range tt.cveIDs { if tt.cveIDs[i] != aCveIDs[i] { t.Errorf("expected cveID: %s, actual %s", tt.cveIDs[i], aCveIDs[i]) } } - if tt.vulnID != aVunlnID { - t.Errorf("expected vulnID: %s, actual %s", tt.vulnID, aVunlnID) + if tt.vulnID != aVulnID { + t.Errorf("expected vulnID: %s, actual %s", tt.vulnID, aVulnID) } } } diff --git a/scan/redhat.go b/scan/redhat.go index 22bdf6818a..53b640e72c 100644 --- a/scan/redhat.go +++ b/scan/redhat.go @@ -112,45 +112,46 @@ func (o *redhat) checkIfSudoNoPasswd() error { // CentOS 6 ... yum-plugin-changelog // CentOS 7 ... yum-plugin-changelog // RHEL, Amazon ... no additinal packages needed -func (o *redhat) install() error { +func (o *redhat) checkDependencies() error { switch o.Distro.Family { case "rhel", "amazon": - o.log.Infof("Nothing to do") + // o.log.Infof("Nothing to do") return nil - } - // CentOS - return o.installYumChangelog() -} -func (o *redhat) installYumChangelog() error { - if o.Distro.Family == "centos" { + case "centos": var majorVersion int if 0 < len(o.Distro.Release) { majorVersion, _ = strconv.Atoi(strings.Split(o.Distro.Release, ".")[0]) } else { - return fmt.Errorf( - "Not implemented yet: %s", o.Distro) + return fmt.Errorf("Not implemented yet: %s", o.Distro) } - var packName = "" + var name = "" if majorVersion < 6 { - packName = "yum-changelog" + name = "yum-changelog" } else { - packName = "yum-plugin-changelog" + name = "yum-plugin-changelog" } - cmd := "rpm -q " + packName + cmd := "rpm -q " + name if r := o.exec(cmd, noSudo); r.isSuccess() { - o.log.Infof("Ignored: %s already installed", packName) return nil } + o.lackDependencies = []string{name} + return nil - o.log.Infof("Installing %s...", packName) - cmd = util.PrependProxyEnv("yum install -y " + packName) + default: + return fmt.Errorf("Not implemented yet: %s", o.Distro) + } +} + +func (o *redhat) install() error { + for _, name := range o.lackDependencies { + cmd := util.PrependProxyEnv("yum install -y " + name) if r := o.exec(cmd, sudo); !r.isSuccess() { return fmt.Errorf("Failed to SSH: %s", r) } - o.log.Infof("Installed: %s", packName) + o.log.Infof("Installed: %s", name) } return nil } @@ -252,7 +253,7 @@ func (o *redhat) scanUnsecurePackages() ([]CvePacksInfo, error) { // For CentOS func (o *redhat) scanUnsecurePackagesUsingYumCheckUpdate() (CvePacksList, error) { - cmd := "LANG=en_US.UTF-8 yum --color=never check-update" + cmd := "LANGUAGE=en_US.UTF-8 yum --color=never check-update" r := o.exec(util.PrependProxyEnv(cmd), sudo) if !r.isSuccess(0, 100) { //returns an exit code of 100 if there are available updates. @@ -408,7 +409,7 @@ func (o *redhat) parseYumCheckUpdateLines(stdout string) (results models.Package func (o *redhat) parseYumCheckUpdateLine(line string) (models.PackageInfo, error) { fields := strings.Fields(line) - if len(fields) != 3 { + if len(fields) < 3 { return models.PackageInfo{}, fmt.Errorf("Unknown format: %s", line) } splitted := strings.Split(fields[0], ".") @@ -548,7 +549,7 @@ func (o *redhat) getAllChangelog(packInfoList models.PackageInfoList) (stdout st } // yum update --changelog doesn't have --color option. - command += fmt.Sprintf(" LANG=en_US.UTF-8 yum update --changelog %s", packageNames) + command += fmt.Sprintf(" LANGUAGE=en_US.UTF-8 yum update --changelog %s", packageNames) r := o.exec(command, sudo) if !r.isSuccess(0, 1) { @@ -590,7 +591,7 @@ func (o *redhat) scanUnsecurePackagesUsingYumPluginSecurity() (CvePacksList, err // get package name, version, rel to be upgrade. // cmd = "yum check-update --security" - cmd = "LANG=en_US.UTF-8 yum --color=never check-update" + cmd = "LANGUAGE=en_US.UTF-8 yum --color=never check-update" r = o.exec(util.PrependProxyEnv(cmd), o.sudo()) if !r.isSuccess(0, 100) { //returns an exit code of 100 if there are available updates. diff --git a/scan/redhat_test.go b/scan/redhat_test.go index 9bd87f2665..60a5df7cd4 100644 --- a/scan/redhat_test.go +++ b/scan/redhat_test.go @@ -616,6 +616,7 @@ Obsoleting Packages python-libs.i686 2.6.6-64.el6 rhui-REGION-rhel-server-releases python-ordereddict.noarch 1.1-3.el6ev installed bind-utils.x86_64 30:9.3.6-25.P1.el5_11.8 updates +pytalloc.x86_64 2.0.7-2.el6 @CentOS 6.5/6.5 ` r.Packages = []models.PackageInfo{ @@ -644,6 +645,11 @@ bind-utils.x86_64 30:9.3.6-25.P1.el5_11.8 updates Version: "1.0", Release: "1", }, + { + Name: "pytalloc", + Version: "2.0.1", + Release: "0", + }, } var tests = []struct { in string @@ -687,6 +693,13 @@ bind-utils.x86_64 30:9.3.6-25.P1.el5_11.8 updates NewVersion: "9.3.6", NewRelease: "25.P1.el5_11.8", }, + { + Name: "pytalloc", + Version: "2.0.1", + Release: "0", + NewVersion: "2.0.7", + NewRelease: "2.el6", + }, }, }, } diff --git a/scan/serverapi.go b/scan/serverapi.go index 78e838068b..bf0efef7d1 100644 --- a/scan/serverapi.go +++ b/scan/serverapi.go @@ -18,7 +18,10 @@ along with this program. If not, see . package scan import ( + "bufio" "fmt" + "os" + "strings" "time" "github.com/Sirupsen/logrus" @@ -40,7 +43,10 @@ type osTypeInterface interface { setDistro(string, string) getDistro() config.Distro - // getFamily() string + + // checkDependencies checks if dependencies are installed on the target server. + checkDependencies() error + getLackDependencies() []string checkIfSudoNoPasswd() error detectPlatform() error @@ -60,7 +66,7 @@ type osTypeInterface interface { setErrs([]error) } -// osPackages included by linux struct +// osPackages is included by base struct type osPackages struct { // installed packages Packages models.PackageInfoList @@ -424,12 +430,65 @@ func detectPlatforms() []error { // Prepare installs requred packages to scan vulnerabilities. func Prepare() []error { - return parallelSSHExec(func(o osTypeInterface) error { + errs := parallelSSHExec(func(o osTypeInterface) error { + if err := o.checkDependencies(); err != nil { + return err + } + return nil + }) + if len(errs) != 0 { + return errs + } + + var targets []osTypeInterface + for _, s := range servers { + deps := s.getLackDependencies() + if len(deps) != 0 { + targets = append(targets, s) + } + } + if len(targets) == 0 { + Log.Info("No need to install dependencies") + return nil + } + + Log.Info("Below servers are needed to install dependencies") + for _, s := range targets { + for _, d := range s.getLackDependencies() { + Log.Infof(" - %s on %s", d, s.getServerInfo().GetServerName()) + } + } + Log.Info("Is this ok to install dependencies on the servers? [y/N]") + + reader := bufio.NewReader(os.Stdin) + for { + text, err := reader.ReadString('\n') + if err != nil { + return []error{err} + } + switch strings.TrimSpace(text) { + case "", "N", "n": + return nil + case "y", "Y": + goto yes + default: + Log.Info("Please enter y or N") + } + } + +yes: + servers = targets + errs = parallelSSHExec(func(o osTypeInterface) error { if err := o.install(); err != nil { return err } return nil }) + if len(errs) != 0 { + return errs + } + Log.Info("All dependencies were installed correctly") + return nil } // Scan scan @@ -450,7 +509,7 @@ func Scan() []error { defer func() { if cache.DB != nil { - defer cache.DB.Close() + cache.DB.Close() } }() diff --git a/setup/docker/README.ja.md b/setup/docker/README.ja.md deleted file mode 100644 index 831060c8d1..0000000000 --- a/setup/docker/README.ja.md +++ /dev/null @@ -1,101 +0,0 @@ -# Vuls on Docker - -## What's Vuls-On-Docker - -- 数個のコマンドを実行するだけでVulsとvulsrepoのセットアップが出来るスクリプト -- Dockerコンテナ上にVulsと[vulsrepo](https://github.com/usiusi360/vulsrepo)をセットアップ可能 -- スキャン結果をvulsrepoでブラウザで分析可能 -- 脆弱性データベースの更新が可能 -- モジュールのアップデートが可能 - -## Setting up your machine - -1. [Install Docker](https://docs.docker.com/engine/installation/) -2. [Install Docker-Compose](https://docs.docker.com/compose/install/) -3. 実行前に以下のコマンドが実行可能なことを確認する - - ``` - $ docker version - $ docker-compose version - ``` - -4. Vulsをgit clone - ``` - mkdir work - cd work - git clone https://github.com/future-architect/vuls.git - cd vuls/setup/docker - ``` - -## Start A Vuls Container - -- 以下のコマンドを実行してコンテナをビルドする - - ``` - $ docker-compose up -d - ``` - -## Setting up Vuls - -1. スキャン対象サーバのSSH秘密鍵を保存(vuls/setup/docker/conf/)する -2. config.toml(vuls/setup/docker/conf/config.toml) を環境に合わせて作成する - - ``` - [servers] - - [servers.172-31-4-82] - host = "172.31.4.82" - user = "ec2-user" - keyPath = "conf/id_rsa" - ``` - -## Fetch Vulnerability database - -- NVDから脆弱性データベースを取得する - ``` - $ docker exec -t vuls scripts/fetch_nvd_all.sh - ``` - -- レポートを日本語化する場合は、JVNから脆弱性データを取得する - ``` - $ docker exec -t vuls scripts/fetch_jvn_all.sh - ``` - -## Scan servers with Vuls-On-Docker - -- スキャンを実行する - - ``` - $ docker exec -t vuls vuls prepare -config=conf/config.toml - $ docker exec -t vuls scripts/scan_for_vulsrepo.sh - ``` - -## See the results in a browser - -``` -http://${Vuls_Host}/vulsrepo/ -``` - -# Update modules - -- vuls, go-cve-dictionary, vulsrepoのモジュールをアップデートする - ``` - $ docker exec -t vuls scripts/update_modules.sh - ``` - -# Update Vulnerability database - -- NVDの過去２年分の脆弱性データベースを更新する - ``` - $ docker exec -t vuls scripts/fetch_nvd_last2y.sh - ``` - -- JVNの過去１ヶ月分の脆弱性データベースを更新する - ``` - $ docker exec -t vuls scripts/fetch_jvn_month.sh - ``` - -- JVNの過去1週間分の脆弱性データベースを更新する - ``` - $ docker exec -t vuls scripts/fetch_jvn_week.sh - ``` diff --git a/setup/docker/README.md b/setup/docker/README.md index 2df2d58d59..d72aae70c2 100644 --- a/setup/docker/README.md +++ b/setup/docker/README.md @@ -1,87 +1,183 @@ -# Vuls on Docker +# Vuls Docker components -## What's Vuls-On-Docker +This is the Git repo of the official Docker image for vuls. -- This is a dockernized-Vuls with vulsrepo UI in it. -- It's designed to reduce the cost of installation and the dependencies that vuls requires. -- You can run install and run Vuls on your machine with only a few commands. -- The result can be viewed with a browser +# Supported tags and respective `Dockerfile` links -## Setting up your machine - -1. [Install Docker](https://docs.docker.com/engine/installation/) -2. [Install Docker-Compose](https://docs.docker.com/compose/install/) -3. Make sure that you can run the following commands before you move on. +- go-cve-dictionary + - [`latest` (*go-cve-dictionary:latest Dockerfile*)]() +- vuls + - [`latest` (*vuls:latest Dockerfile*)]() +- vulsrepo + - [`latest` (*vulsrepo:latest Dockerfile*)]() - ``` - $ docker version - $ docker-compose version - ``` - -4. git clone vuls - ``` - mkdir work - cd work - git clone https://github.com/future-architect/vuls.git - cd vuls/setup/docker - ``` +This image version is same as the github repository version. +# Caution +This image is built per commit. +If you want to use the latest docker image, you should remove the existing image, and pull it once again. -## Start A Vuls Container +1. Confirm your vuls version -- Execute the following command to build and run a Vuls Container +- go-cve-dictionary - ``` - $ docker-compose up -d - ``` +```console +$ docker run --rm vuls/go-cve-dictionary -v -## Setting up Vuls +go-cve-dictionary v0.0.xxx xxxx +``` + +- vuls + +```console +$ docker run --rm vuls/vuls -v + +vuls v0.0.xxx xxxx +``` + +2. Remove your old docker images + +- go-cve-dictionary + +``` +$ docker rmi vuls/go-cve-dictionary +``` + +``` +$ docker rmi vuls/vuls +``` + +- vuls + +``` +$ docker rmi vuls/vuls +``` + +3. Pull new vuls docker images + +- go-cve-dictionary + +``` +$ docker pull vuls/go-cve-dictionary +``` + +- vuls -1. Locate ssh-keys of target servers in (vuls/setup/docker/conf/) -2. Create and adjust config.toml(vuls/setup/docker/conf/config.toml) to your environment - - ``` - [servers] +``` +$ docker pull vuls/vuls +``` + +4. Confirm your vuls version + +```console +$ docker run --rm vuls/go-cve-dictionary -v + +go-cve-dictionary v0.1.xxx xxxx +``` - [servers.172-31-4-82] - host = "172.31.4.82" - user = "ec2-user" - keyPath = "conf/id_rsa" - ``` +- vuls -## Fetch Vulnerability database +```console +$ docker run --rm vuls/vuls -v -- Fetch Vulnerability database from NVD - ``` - $ docker exec -t vuls scripts/fetch_nvd_all.sh - ``` +vuls v0.1.xxx xxxx +``` + + +# How to use this image -## Scan servers with Vuls-On-Docker +1. fetch nvd (vuls/go-cve-dictionary) +1. configuration (vuls/vuls) +1. prepare (vuls/vuls) +1. scan (vuls/vuls) +1. vulsrepo (vuls/vulsrepo) + +## Step1. Fetch NVD + +```console +$ for i in {2002..2016}; do \ + docker run --rm -it \ + -v $PWD:/vuls \ + -v $PWD/go-cve-dictionary-log:/var/log/vuls \ + vuls/go-cve-dictionary fetchnvd -years $i; \ + done +``` -- Use the embedded script to scan servers for vulsrepo(or run whatever with docker exec) +## Step2. Configuration - ``` - $ docker exec -t vuls vuls prepare -config=conf/config.toml - $ docker exec -t vuls scripts/scan_for_vulsrepo.sh - ``` +Create config.toml referring to [this](https://github.com/future-architect/vuls#configuration). -## See the results in a browser +```toml +[servers] +[servers.amazon] +host = "54.249.93.16" +port = "22" +user = "vuls-user" +keyPath = "/root/.ssh/id_rsa" # path to ssh private key in docker ``` -http://${Vuls_Host}/vulsrepo/ +　 + +```console +$ docker run --rm \ + -v ~/.ssh:/root/.ssh:ro \ + -v $PWD:/vuls \ + -v $PWD/vuls-log:/var/log/vuls \ + vuls/vuls configtest \ + -config=./config.toml # path to config.toml in docker ``` -# Update modules +## Step3. Prepare + +```console +$ docker run --rm \ + -v ~/.ssh:/root/.ssh:ro \ + -v $PWD:/vuls \ + -v $PWD/vuls-log:/var/log/vuls \ + vuls/vuls prepare \ + -config=./config.toml # path to config.toml in docker +``` + +## Step4. Scan + +```console +$ docker run --rm -it \ + -v ~/.ssh:/root/.ssh:ro \ + -v $PWD:/vuls \ + -v $PWD/vuls-log:/var/log/vuls \ + -v /etc/localtime:/etc/localtime:ro \ + -e "TZ=Asia/Tokyo" \ + vuls/vuls scan \ + -cve-dictionary-dbpath=/vuls/cve.sqlite3 \ + -report-json \ + -config=./config.toml # path to config.toml in docker +``` + +## Step5. vulsrepo + +```console +$docker run -dt \ + -v $PWD:/vuls \ + -p 80:80 \ + vuls/vulsrepo +``` + +# User Feedback + +## Documentation + +Documentation for this image is stored in the [`docker/` directory]() of the [`future-architect/vuls` GitHub repo](https://github.com/future-architect/vuls). + +## Issues -- update vuls, go-cve-dictionary, vulsrepo - ``` - $ docker exec -t vuls scripts/update_modules.sh - ``` +If you have any problems with or questions about this image, please contact us through a [GitHub issue](https://github.com/future-architect/vuls/issues). -# Update Vulnerability database +## Contributing -- Fetch Vulnerability database from NVD - ``` - $ docker exec -t vuls scripts/fetch_nvd_last2y.sh - ``` +1. fork a repository: github.com/future-architect/vuls to github.com/you/repo +1. get original code: go get github.com/future-architect/vuls +1. work on original code +1. add remote to your repo: git remote add myfork https://github.com/you/repo.git +1. push your changes: git push myfork +1. create a new Pull Request diff --git a/setup/docker/conf/.gitkeep b/setup/docker/conf/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/setup/docker/docker-compose.yml b/setup/docker/docker-compose.yml deleted file mode 100644 index cb749d1d40..0000000000 --- a/setup/docker/docker-compose.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: '2' -services: - vuls: - container_name: vuls - build: ./dockerfile - image: vuls-docker:0.1 - volumes: - - ./conf:/opt/vuls/conf - ports: - - "80:80" - diff --git a/setup/docker/dockerfile/Dockerfile b/setup/docker/dockerfile/Dockerfile deleted file mode 100644 index 625658eb51..0000000000 --- a/setup/docker/dockerfile/Dockerfile +++ /dev/null @@ -1,73 +0,0 @@ -FROM buildpack-deps:jessie-scm - -# golang Install -RUN apt-get update && apt-get install -y --no-install-recommends \ - g++ \ - gcc \ - libc6-dev \ - make \ - curl \ - && rm -rf /var/lib/apt/lists/* - -ENV GOLANG_VERSION 1.6.2 -ENV GOLANG_DOWNLOAD_URL https://golang.org/dl/go$GOLANG_VERSION.linux-amd64.tar.gz -ENV GOLANG_DOWNLOAD_SHA256 e40c36ae71756198478624ed1bb4ce17597b3c19d243f3f0899bb5740d56212a - -RUN curl -fsSL "$GOLANG_DOWNLOAD_URL" -o golang.tar.gz \ - && echo "$GOLANG_DOWNLOAD_SHA256 golang.tar.gz" | sha256sum -c - \ - && tar -C /usr/local -xzf golang.tar.gz \ - && rm golang.tar.gz - -ENV GOPATH /go -ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH - -RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" - -# glide install -ENV GLIDE_VERSION 0.10.2 -ENV GLIDE_DOWNLOAD_URL https://github.com/Masterminds/glide/releases/download/$GLIDE_VERSION/glide-$GLIDE_VERSION-linux-amd64.tar.gz -RUN curl -fsSL "$GLIDE_DOWNLOAD_URL" -o glide.tar.gz \ - && mkdir /usr/local/glide \ - && tar -C /usr/local/glide -xzf glide.tar.gz \ - && ln -s /usr/local/glide/linux-amd64/glide /usr/local/bin/ \ - && rm glide.tar.gz - -#Vuls Install -ENV VULS_ROOT /opt/vuls -RUN mkdir -p /var/log/vuls ${VULS_ROOT}/conf /root/.ssh/ -RUN chmod 700 -R /var/log/vuls $VULS_ROOT -# RUN go get github.com/kotakanbe/go-cve-dictionary -# RUN go get github.com/future-architect/vuls - -RUN go get -v -d github.com/kotakanbe/go-cve-dictionary \ - && cd $GOPATH/src/github.com/kotakanbe/go-cve-dictionary \ - && glide install \ - && go install - -RUN go get -v -d github.com/future-architect/vuls \ - && cd $GOPATH/src/github.com/future-architect/vuls \ - && glide install \ - && go install - -# Copy custom Scripts -COPY ./scripts/ ${VULS_ROOT}/scripts -RUN chmod 755 ${VULS_ROOT}/scripts/* - - -#Vulrepo Install -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - apache2 \ - libcgi-pm-perl \ - libjson-perl \ - && rm -rf /var/lib/apt/lists/* \ - && cd /var/www/html/ \ - && git clone https://github.com/usiusi360/vulsrepo \ - && mkdir /var/www/html/vulsrepo/results \ - && cp /var/www/html/vulsrepo/dist/cgi/vulsrepo.conf.sample /etc/apache2/conf-enabled/vulsrepo.conf \ - && a2enmod cgid - -#Home -WORKDIR /opt/vuls -EXPOSE 80 443 -ENTRYPOINT service apache2 start && tail -f /dev/null diff --git a/setup/docker/dockerfile/scripts/fetch_jvn_all.sh b/setup/docker/dockerfile/scripts/fetch_jvn_all.sh deleted file mode 100644 index f738a10b45..0000000000 --- a/setup/docker/dockerfile/scripts/fetch_jvn_all.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -VULS_ROOT=/opt/vuls -#VULS_CONF=${VULS_ROOT}/conf -cd $VULS_ROOT -for i in {2002..2016}; do go-cve-dictionary fetchjvn -years $i; done - diff --git a/setup/docker/dockerfile/scripts/fetch_jvn_last2y.sh b/setup/docker/dockerfile/scripts/fetch_jvn_last2y.sh deleted file mode 100644 index 62a0d992d9..0000000000 --- a/setup/docker/dockerfile/scripts/fetch_jvn_last2y.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -VULS_ROOT=/opt/vuls -#VULS_CONF=${VULS_ROOT}/conf -cd $VULS_ROOT -go-cve-dictionary fetchjvn -last2y - diff --git a/setup/docker/dockerfile/scripts/fetch_jvn_latest.sh b/setup/docker/dockerfile/scripts/fetch_jvn_latest.sh deleted file mode 100644 index 66a5b36f80..0000000000 --- a/setup/docker/dockerfile/scripts/fetch_jvn_latest.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -VULS_ROOT=/opt/vuls -#VULS_CONF=${VULS_ROOT}/conf -cd $VULS_ROOT -go-cve-dictionary fetchjvn -latest diff --git a/setup/docker/dockerfile/scripts/fetch_nvd_all.sh b/setup/docker/dockerfile/scripts/fetch_nvd_all.sh deleted file mode 100644 index fb9060d104..0000000000 --- a/setup/docker/dockerfile/scripts/fetch_nvd_all.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -VULS_ROOT=/opt/vuls -#VULS_CONF=${VULS_ROOT}/conf -cd $VULS_ROOT -for i in {2002..2016}; do go-cve-dictionary fetchnvd -years $i; done - diff --git a/setup/docker/dockerfile/scripts/fetch_nvd_last2y.sh b/setup/docker/dockerfile/scripts/fetch_nvd_last2y.sh deleted file mode 100644 index 57a2955830..0000000000 --- a/setup/docker/dockerfile/scripts/fetch_nvd_last2y.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -VULS_ROOT=/opt/vuls -#VULS_CONF=${VULS_ROOT}/conf -cd $VULS_ROOT -go-cve-dictionary fetchnvd -last2y - diff --git a/setup/docker/dockerfile/scripts/scan_for_vulsrepo.sh b/setup/docker/dockerfile/scripts/scan_for_vulsrepo.sh deleted file mode 100644 index c3c43161c2..0000000000 --- a/setup/docker/dockerfile/scripts/scan_for_vulsrepo.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -VULS_ROOT=/opt/vuls -VULS_CONF=${VULS_ROOT}/conf -APACHE_VULSREPO_ROOT=/var/www/html/vulsrepo -cd $VULS_ROOT -vuls scan -report-json --cve-dictionary-dbpath=${VULS_ROOT}/cve.sqlite3 -config=${VULS_CONF}/config.toml -rm ${APACHE_VULSREPO_ROOT}/results/* -cp ${VULS_ROOT}/results/current/* ${APACHE_VULSREPO_ROOT}/results diff --git a/setup/docker/dockerfile/scripts/update_modules.sh b/setup/docker/dockerfile/scripts/update_modules.sh deleted file mode 100644 index 9cd3b493dd..0000000000 --- a/setup/docker/dockerfile/scripts/update_modules.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -cd $GOPATH/src/github.com/future-architect/vuls -git pull origin master -glide install -go install - - -cd $GOPATH/src/github.com/kotakanbe/go-cve-dictionary -git pull origin master -glide install -go install - - -cd /var/www/html/vulsrepo -git pull origin master diff --git a/setup/docker/go-cve-dictionary/latest/Dockerfile b/setup/docker/go-cve-dictionary/latest/Dockerfile new file mode 100644 index 0000000000..c201bec3ff --- /dev/null +++ b/setup/docker/go-cve-dictionary/latest/Dockerfile @@ -0,0 +1,19 @@ +FROM golang:latest + +MAINTAINER hikachan sadayuki-matsuno + +ENV REPOSITORY github.com/kotakanbe/go-cve-dictionary +ENV LOGDIR /var/log/vuls +ENV WORKDIR /vuls +# go-cve-dictionary install +RUN git clone https://$REPOSITORY.git $GOPATH/src/$REPOSITORY \ + && cd $GOPATH/src/$REPOSITORY \ + && make install \ + && mkdir -p $LOGDIR + +VOLUME [$WORKDIR, $LOGDIR] +WORKDIR $WORKDIR +ENV PWD $WORKDIR + +ENTRYPOINT ["go-cve-dictionary"] +CMD ["--help"] diff --git a/setup/docker/go-cve-dictionary/latest/README.md b/setup/docker/go-cve-dictionary/latest/README.md new file mode 100644 index 0000000000..588294ae8b --- /dev/null +++ b/setup/docker/go-cve-dictionary/latest/README.md @@ -0,0 +1,89 @@ +# go-cve-dictionary-Docker + +This is the Git repo of the official Docker image for go-cve-dictionary. +See the [Hub page](https://hub.docker.com/r/vuls/go-cve-dictionary/) for the full readme on how to use the Docker image and for information regarding contributing and issues. + +# Supported tags and respective `Dockerfile` links + +- [`latest` (*go-cve-dictionary:latest Dockerfile*)](https://github.com/future-architect/vuls/blob/master/setup/docker/go-cve-dictionary/latest/Dockerfile) + +# Caution + +This image is built per commit. +If you want to use the latest docker image, you should remove the existing image, and pull it once again. + +- Remove old docker image + +``` +$ docker rmi vuls/go-cve-dictionary +``` + +- Pull new docker image + +``` +$ docker pull vuls/go-cve-dictionary +``` + +# What is go-cve-dictionary? + +This is tool to build a local copy of the NVD (National Vulnerabilities Database) [1] and the Japanese JVN [2], which contain security vulnerabilities according to their CVE identifiers [3] including exhaustive information and a risk score. The local copy is generated in sqlite format, and the tool has a server mode for easy querying. + +[1] https://en.wikipedia.org/wiki/National_Vulnerability_Database +[2] https://en.wikipedia.org/wiki/Common_Vulnerabilities_and_Exposures +[3] http://jvndb.jvn.jp/apis/termsofuse.html + +# How to use this image + +## check vuls version + +``` +$ docker run --rm vuls/go-cve-dictionary -v +``` + +## fetchnvd + +```console +$ for i in {2002..2016}; do \ + docker run --rm -it \ + -v $PWD:/vuls \ + -v $PWD/go-cve-dictionary-log:/var/log/vuls \ + vuls/go-cve-dictionary fetchnvd -years $i; \ + done +``` + +## server + +```console +$ docker run -dt \ + --name go-cve-dictionary \ + -v $PWD:/vuls \ + -v $PWD/go-cve-dictionary-log:/var/log/vuls \ + --expose 1323 \ + -p 1323:1323 \ + vuls/go-cve-dictionary server --bind=0.0.0.0 +``` + +Prease refer to [this](https://hub.docker.com/r/vuls/go-cve-dictionary). + +## vuls + +Please refer to [this](https://hub.docker.com/r/vuls/vuls/). + +# User Feedback + +## Documentation + +Documentation for this image is stored in the [`docker/` directory](https://github.com/future-architect/vuls/tree/master/setup/docker) of the [`future-architect/vuls` GitHub repo](https://github.com/future-architect/vuls). + +## Issues + +If you have any problems with or questions about this image, please contact us through a [GitHub issue](https://github.com/future-architect/vuls/issues). + +## Contributing + +1. fork a repository: github.com/future-architect/vuls to github.com/you/repo +1. get original code: go get github.com/future-architect/vuls +1. work on original code +1. add remote to your repo: git remote add myfork https://github.com/you/repo.git +1. push your changes: git push myfork +1. create a new Pull Request diff --git a/setup/docker/vuls/latest/Dockerfile b/setup/docker/vuls/latest/Dockerfile new file mode 100644 index 0000000000..1db73c550f --- /dev/null +++ b/setup/docker/vuls/latest/Dockerfile @@ -0,0 +1,19 @@ +FROM golang:latest + +MAINTAINER hikachan sadayuki-matsuno + +ENV REPOSITORY github.com/future-architect/vuls +ENV LOGDIR /var/log/vuls +ENV WORKDIR /vuls +# go-cve-dictionary install +RUN git clone https://$REPOSITORY.git $GOPATH/src/$REPOSITORY \ + && cd $GOPATH/src/$REPOSITORY \ + && make install \ + && mkdir -p $LOGDIR + +VOLUME [$WORKDIR, $LOGDIR] +WORKDIR $WORKDIR +ENV PWD $WORKDIR + +ENTRYPOINT ["vuls"] +CMD ["--help"] diff --git a/setup/docker/vuls/latest/README.md b/setup/docker/vuls/latest/README.md new file mode 100644 index 0000000000..0fe862850e --- /dev/null +++ b/setup/docker/vuls/latest/README.md @@ -0,0 +1,121 @@ +# Vuls-Docker + +This is the Git repo of the official Docker image for vuls. +See the [Hub page](https://hub.docker.com/r/vuls/vuls/) for the full readme on how to use the Docker image and for information regarding contributing and issues. + +# Supported tags and respective `Dockerfile` links + +- [`latest` (*vuls:latest Dockerfile*)](https://github.com/future-architect/vuls/blob/master/setup/docker/vuls/latest/Dockerfile) + +# Caution + +This image is built per commit. +If you want to use the latest docker image, you should remove the existing image, and pull it once again. + +- Remove old docker image + +``` +$ docker rmi vuls/vuls +``` + +- Pull new docker image + +``` +$ docker pull vuls/vuls +``` + +# What is Vuls? + +Vuls is the Vulnerability scanner for Linux/FreeBSD, agentless, written in golang. +Please see the [Documentation](https://github.com/future-architect/vuls) + +![logo](https://github.com/future-architect/vuls/blob/master/img/vuls_logo.png?raw=true) + +# How to use this image + +## check vuls version + +``` +$ docker run --rm vuls/vuls -v +``` + +## configtest + +Create config.toml referring to [this](https://github.com/future-architect/vuls#configuration). + +```toml +[servers] + +[servers.amazon] +host = "54.249.93.16" +port = "22" +user = "vuls-user" +keyPath = "/root/.ssh/id_rsa" # path to ssh private key in docker +``` +　 + +```console +$ docker run --rm \ + -v ~/.ssh:/root/.ssh:ro \ + -v $PWD:/vuls \ + -v $PWD/vuls-log:/var/log/vuls \ + vuls/vuls configtest +``` + + +## prepare + +```console +$ docker run --rm \ + -v ~/.ssh:/root/.ssh:ro \ + -v $PWD:/vuls \ + -v $PWD/vuls-log:/var/log/vuls \ + vuls/vuls prepare \ + -config=./config.toml # path to config.toml in docker +``` + +## scan + +```console +$ docker run --rm -it \ + -v ~/.ssh:/root/.ssh:ro \ + -v $PWD:/vuls \ + -v $PWD/vuls-log:/var/log/vuls \ + -v /etc/localtime:/etc/localtime:ro \ + vuls/vuls scan \ + -cve-dictionary-dbpath=/vuls/cve.sqlite3 \ + -config=./config.toml \ # path to config.toml in docker + -report-json +``` + +## tui + +```console +$ docker run --rm -it \ + -v $PWD:/vuls \ + -v $PWD/vuls-log:/var/log/vuls \ + vuls/vuls tui +``` + +## vulsrepo + +Prease refer to [this](https://hub.docker.com/r/vuls/vulsrepo/). + +# User Feedback + +## Documentation + +Documentation for this image is stored in the [`docker/` directory](https://github.com/future-architect/vuls/tree/master/setup/docker) of the [`future-architect/vuls` GitHub repo](https://github.com/future-architect/vuls). + +## Issues + +If you have any problems with or questions about this image, please contact us through a [GitHub issue](https://github.com/future-architect/vuls/issues). + +## Contributing + +1. fork a repository: github.com/future-architect/vuls to github.com/you/repo +1. get original code: go get github.com/future-architect/vuls +1. work on original code +1. add remote to your repo: git remote add myfork https://github.com/you/repo.git +1. push your changes: git push myfork +1. create a new Pull Request diff --git a/setup/docker/vulsrepo/latest/Dockerfile b/setup/docker/vulsrepo/latest/Dockerfile new file mode 100644 index 0000000000..8410553837 --- /dev/null +++ b/setup/docker/vulsrepo/latest/Dockerfile @@ -0,0 +1,31 @@ +FROM httpd:2.4 + +MAINTAINER hikachan sadayuki-matsuno +# install packages +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + vim \ + git \ + libcgi-pm-perl \ + libjson-perl \ + && rm -r /var/lib/apt/lists/* + +# env +ENV HTTPD_PREFIX /usr/local/apache2 + +VOLUME /vuls + +WORKDIR ${HTTPD_PREFIX}/htdocs +RUN git clone https://github.com/usiusi360/vulsrepo.git \ + && echo "LoadModule cgid_module modules/mod_cgid.so" >> $HTTPD_PREFIX/conf/httpd.conf \ + && echo "" >> $HTTPD_PREFIX/conf/httpd.conf \ + && echo " Options +ExecCGI +FollowSymLinks" >> $HTTPD_PREFIX/conf/httpd.conf \ + && echo " AddHandler cgi-script cgi" >> $HTTPD_PREFIX/conf/httpd.conf \ + && echo "" >> $HTTPD_PREFIX/conf/httpd.conf \ + && sed -i -e 's/User daemon/#User/g' $HTTPD_PREFIX/conf/httpd.conf \ + && sed -i -e 's/Group daemon/#Group/g' $HTTPD_PREFIX/conf/httpd.conf \ + && ln -snf /vuls/results /usr/local/apache2/htdocs/vulsrepo/results + +EXPOSE 80 +CMD ["httpd-foreground"] diff --git a/setup/docker/vulsrepo/latest/README.md b/setup/docker/vulsrepo/latest/README.md new file mode 100644 index 0000000000..7b5ff687b6 --- /dev/null +++ b/setup/docker/vulsrepo/latest/README.md @@ -0,0 +1,47 @@ +# VulsRepo-Docker + +This is the Git repo of the official Docker image for vulsrepo. +See the [Hub page](https://hub.docker.com/r/vuls/vulsrepo/) for the full readme on how to use the Docker image and for information regarding contributing and issues. + +# Supported tags and respective `Dockerfile` links + +- [`latest` (*vulsrepo:latest Dockerfile*)](https://github.com/future-architect/vuls/blob/master/setup/docker/vulsrepo/latest/Dockerfile) + +# Caution + +This image is built per commit. +If you want to use the latest docker image, you should remove the existing image, and pull it once again. + +# What is vulsrepo? + +VulsRepo is visualized based on the json report output in [vuls](https://github.com/future-architect/vuls). + +# How to use this image + +## vulsrepo + +```console +$docker run -dt \ + -v $PWD:/vuls \ + -p 80:80 \ + vuls/vulsrepo +``` + +# User Feedback + +## Documentation + +Documentation for this image is stored in the [`docker/` directory](https://github.com/future-architect/vuls/tree/master/setup/docker) of the [`future-architect/vuls` GitHub repo](https://github.com/future-architect/vuls). + +## Issues + +If you have any problems with or questions about this image, please contact us through a [GitHub issue](https://github.com/future-architect/vuls/issues). + +## Contributing + +1. fork a repository: github.com/future-architect/vuls to github.com/you/repo +1. get original code: go get github.com/future-architect/vuls +1. work on original code +1. add remote to your repo: git remote add myfork https://github.com/you/repo.git +1. push your changes: git push myfork +1. create a new Pull Request diff --git a/version/version.go b/version/version.go deleted file mode 100644 index 3ba04ed2be..0000000000 --- a/version/version.go +++ /dev/null @@ -1,24 +0,0 @@ -/* Vuls - Vulnerability Scanner -Copyright (C) 2016 Future Architect, Inc. Japan. - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ - -package version - -// Name is Vuls -const Name string = "vuls" - -// Version of Vuls -const Version string = "0.1.6"