Dryrun

.gitignore

@@ -1,3 +1,7 @@
 *.pyc
 *.swp
 build
+.project
+.pydevproject
+logs
+

Makefile

@@ -7,10 +7,10 @@ else
   Q = @
 endif
 
-all: eggs $(ROOTFS) check_convention
+all: $(ROOTFS) check_convention
 
 check_convention:
-	pep8 py --max-line-length=109
+	pep8 py --max-line-length=150
 
 submit:
 	sudo -E solvent submitproduct rootfs $(ROOTFS)
@@ -22,27 +22,28 @@ approve:
 	sudo -E solvent approve --product=rootfs
 
 clean:
-	sudo rm -fr build
-
-eggs: build/master.egg
-
-build/master.egg:
-	$(Q)mkdir -p build
-	PYTHONPATH=py UPSETO_JOIN_PYTHON_NAMESPACES=yes python -m upseto.packegg --entryPoint=py/rackattack/dryrun/master/main.py --output=$@ --createDeps=$@.deps --takeSitePackages --joinPythonNamespaces
--include build/master.egg.deps
+	@sudo rm -fr build
+	@find -name "*.pyc" -delete
+
 
 $(ROOTFS): build/smartctl
 	-sudo mv $(ROOTFS) $(ROOTFS).tmp
 	echo "Bringing source"
 	-mkdir $(@D)
-	sudo -E solvent bring --repositoryBasename=rootfs-basic --product=rootfs --destination=$(ROOTFS).tmp
+	sudo -E solvent bring --repositoryBasename=rootfs-centos7-basic --product=rootfs --destination=$(ROOTFS).tmp
 	sudo chroot $(ROOTFS).tmp yum install $(RPMS_TO_INSTALL) --assumeyes
 	sudo mkdir $(ROOTFS).tmp/usr/share/inaugurator
 	sudo cp ../inaugurator/build/inaugurator.thin.initrd.img ../inaugurator/build/inaugurator.vmlinuz $(ROOTFS).tmp/usr/share/inaugurator
 	sudo chmod 644 /usr/share/inaugurator/*
 	sudo cp ../inaugurator/dist/inaugurator-1.0-py2.7.egg $(ROOTFS).tmp/tmp
 	sudo chroot $(ROOTFS).tmp easy_install /tmp/inaugurator-1.0-py2.7.egg
 	sudo chroot $(ROOTFS).tmp yum install --assumeyes $(YUMCACHE)/mirrors.kernel.org/fedora-epel/7/x86_64/m/msr-tools-1.3-1.el7.x86_64.rpm
+	sudo chroot $(ROOTFS).tmp yum install --assumeyes $(YUMCACHE)/mirrors.kernel.org/fedora-epel/7/x86_64/v/vconfig-1.9-16.el7.x86_64.rpm
+	sudo chroot $(ROOTFS).tmp yum install --assumeyes $(YUMCACHE)/mirrors.kernel.org/fedora-epel/7/x86_64/p/pigz-2.3.1-1.el7.x86_64.rpm
+	sudo chroot $(ROOTFS).tmp pip install rpdb
+	sudo chroot $(ROOTFS).tmp pip install ipaddr
+	sudo sh -c "echo 'MaxSessions 300' >> $(ROOTFS).tmp/etc/ssh/sshd_config"
+	sudo sh -c "echo 'UseDNS no' >> $(ROOTFS).tmp/etc/ssh/sshd_config"
 	sudo cp $< $(ROOTFS).tmp/usr/sbin/
 	sudo rm -fr $(ROOTFS).tmp/tmp/*
 	sudo mv $(ROOTFS).tmp $(ROOTFS)
@@ -69,5 +70,7 @@ RPMS_TO_INSTALL = \
 	strace \
 	zip \
 	tcpdump \
-	unzip
+	unzip \
+	pciutils \
+	nmap-ncat
 

py/rackattack/dryrun/common/__init__.py

@@ -0,0 +1,2 @@
+import upseto.pythonnamespacejoin
+__path__.extend(upseto.pythonnamespacejoin.join(globals()))

py/rackattack/dryrun/common/waitforpredicate.py

@@ -0,0 +1,18 @@
+import time
+import logging
+
+
+class WaitForPredicate:
+
+    def __init__(self, timeout=3, interval=0.1):
+        self._timeout = timeout
+        self._interval = interval
+
+    def waitAndReturn(self, predicate, * args, ** kwargs):
+        before = time.time()
+        while time.time() - before < self._timeout:
+            ret = predicate(* args, ** kwargs)
+            if ret:
+                return ret
+            time.sleep(self._interval)
+        raise Exception("Predicate '%s' did not happen within timeout" % predicate)

py/rackattack/dryrun/dryrunhost.py

@@ -0,0 +1,61 @@
+from rackattack.ssh import connection
+from strato.racktest.hostundertest import plugins
+
+import strato.racktest.hostundertest.builtinplugins.rpm
+import strato.racktest.hostundertest.builtinplugins.seed
+from rackattack import ssh
+import paramiko
+
+from rackattack.ssh import ftp
+from rackattack.ssh import run
+from rackattack.ssh import dirftp
+from rackattack.ssh import tunnel
+
+
+class DryRunHost(object):
+
+    def __init__(self, node, credentials):
+        self.name = node.name()
+        self.ssh = ProxySSHConnection(node.masterHost, node.ipAddress(), credentials)
+        self.__plugins = {}
+        self.node = node
+
+    def __getattr__(self, name):
+        if name not in self.__plugins:
+            self.__plugins[name] = plugins.plugins[name](self)
+        return self.__plugins[name]
+
+
+class ProxySSHConnection(object):
+
+    def __init__(self, masterHost, destIp, credentials):
+        self._masterHost = masterHost
+        self._destIp = destIp
+        self._credentials = credentials
+        self._sshClient = None
+
+    @property
+    def run(self):
+        return run.Run(self._sshClient)
+
+    @property
+    def ftp(self):
+        return ftp.FTP(self._sshClient)
+
+    @property
+    def dirFTP(self):
+        return dirftp.DirFTP(self._sshClient)
+
+    def close(self):
+        self._sshClient.close()
+        self._sshClient = None
+
+    def connect(self):
+        transport = self._masterHost.ssh._sshClient.get_transport()
+        dst = (self._destIp, 22)
+        src = ('127.0.0.1', 0)
+        commChannel = transport.open_channel("direct-tcpip", dst, src)
+        self._sshClient = paramiko.client.SSHClient()
+        self._sshClient.known_hosts = None
+        self._sshClient.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        self._sshClient.connect(src[0], port=src[1], sock=commChannel, **self._credentials)

py/rackattack/dryrun/healthchecher.py

@@ -0,0 +1,94 @@
+import logging
+from rackattack.dryrun.common import waitforpredicate
+from strato.common.multithreading import concurrently
+import pprint
+import servertestresult
+import threading
+import sys
+
+
+def _verifyVmxEnabledByBios(host, resultObject):
+    IA32_FEATURE_CONTROL = '0x3a'
+    VMXON_BIT = 2
+    LOCK_BIT = 0
+    VMX_ENABLED = ((1 << VMXON_BIT) | (1 << LOCK_BIT))
+    regvalue = int(host.kernel.rdmsr(IA32_FEATURE_CONTROL))
+    result = True
+    log = ''
+    if(regvalue & VMX_ENABLED) != VMX_ENABLED:
+        log = "VMX is not enabled in bios register val %(regvalue)x" % dict(regvalue=int(regvalue))
+        result = False
+    resultObject.addCheck('virt', 'virtualization bios', result, log)
+
+
+def _verifyVirtualizationEnabled(host, resultObject):
+    info = host.kernel.cpuinfo()
+    output = ''
+    result = True
+    if not info.hasVt():
+        result = False
+        output = "Virtualization is not supported on %(hostname)s cpuninfo %(cpuinfo)s" % dict(
+            hostname=host.name, cpuinfo=pprint.pprint(info))
+    resultObject.addCheck('virt', 'virtualization cpu support', result, output)
+    if info.hasFlag(0, 'vmx'):
+        _verifyVmxEnabledByBios(host, resultObject)
+
+
+def _checkDisk(hostToCheck, resultObject):
+    result, output = hostToCheck.disk.smartctlStatus('/dev/sda')
+    resultObject.addCheck('disk', 'smartctl /dev/sda', result, output)
+    resultObject.addCheck('disk', 'SSD /dev/sda', not hostToCheck.disk.rotational('sda'))
+
+
+def _pingScript(ip, deviceName):
+    return "ping -c 2 %(ip)s -I %(device)s" % dict(ip=ip, device=deviceName)
+
+
+def _runPing(srcHost, dstHost, netName, testResult, lock):
+    ipDst = dstHost.network.networks[netName]['ip']
+    srcDevice = srcHost.network.networks[netName]['device']
+    log = ''
+    pingScript = _pingScript(ipDst, srcDevice)
+    try:
+        srcHost.ssh.run.script(pingScript)
+        lock.acquire()
+        testResult.addCheck('net', 'ping on %(netName)s from %(src)s to %(dest)s "%(script)s"' %
+                            dict(netName=netName, src=srcHost.name, dest=dstHost.name, script=pingScript),
+                            True, '', (netName, srcHost.name, dstHost.name))
+        lock.release()
+    except:
+        log = "Failed pinging from host %(srchost)s to %(dstHost)s to ip %(ip)s" % dict(
+            srchost=srcHost.name, dstHost=dstHost.name, ip=ipDst)
+        lock.acquire()
+        testResult.addCheck('net', 'ping on %(netName)s from %(src)s to %(dest)s "%(script)s" exception %(exception)s' %
+                            dict(netName=netName, src=srcHost.name, dest=dstHost.name, script=pingScript, exception=sys.exc_info()[1].message),
+                            False, log, (netName, srcHost.name, dstHost.name))
+        lock.release()
+
+
+def _checkNetwork(node1, node2, vlanTags, testResult, lock):
+    for netName in ['untaged'] + vlanTags:
+        _runPing(node1, node2, netName, testResult, lock)
+        _runPing(node2, node1, netName, testResult, lock)
+
+
+def checkServer(serverToCheck, serversToCheckNetwork, testResult, vlanTags):
+    logging.info("Going to check %(server)s", dict(server=serverToCheck.name))
+    _verifyVirtualizationEnabled(serverToCheck, testResult)
+    _checkDisk(serverToCheck, testResult)
+    lock = threading.Lock()
+    jobs = {server.name: (_checkNetwork, server, serverToCheck, vlanTags, testResult, lock)
+            for server in serversToCheckNetwork}
+    concurrently.run(jobs, numberOfThreads=30)
+    return testResult
+
+
+def _partnerServer(masterHost, serversToCheck, serverToCheck):
+    return [masterHost] + [server for server in serversToCheck if server is not serverToCheck]
+
+
+def checkServers(masterHost, hostsToProced, vlanTags):
+    allHosts = [host['host'] for host in hostsToProced]
+    jobs = {host['name']: (checkServer, host['host'], _partnerServer(masterHost, allHosts, host['host']), host['result'], vlanTags)
+            for host in hostsToProced}
+    concurrently.run(jobs, numberOfThreads=10)

py/rackattack/dryrun/lib/cpuinfo.py

@@ -0,0 +1,19 @@
+class CpuInfo(dict):
+
+    def __init__(self, cpuinfoString):
+        processor = {}
+        for cpuinfoLine in cpuinfoString.split('\n'):
+            if len(cpuinfoLine.strip()) == 0:
+                if len(processor.keys()) == 0:
+                    continue
+                self[int(processor['processor'])] = processor
+                processor = {}
+                continue
+            (k, v) = cpuinfoLine.split(':')
+            processor[k.strip()] = v.strip()
+
+    def hasFlag(self, processorNum, flagName):
+        return flagName in self[processorNum]['flags']
+
+    def hasVt(self, processorNum=0):
+        return self.hasFlag(processorNum, 'vmx') or self.hasFlag(processorNum, 'svm')