538 lines
19 KiB
Python
538 lines
19 KiB
Python
# Copyright (c) 2012, 2013, Oracle and/or its affiliates. All rights reserved.
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; version 2 of the License.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, write to the Free Software
|
|
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
|
|
|
|
"""Tools for performing tasks on (possibly remote) Cluster hosts."""
|
|
|
|
import time
|
|
import posixpath
|
|
import ntpath
|
|
import abc
|
|
import os
|
|
import socket
|
|
import subprocess
|
|
import tempfile
|
|
import shutil
|
|
import logging
|
|
import platform
|
|
import json
|
|
import stat
|
|
|
|
import util
|
|
|
|
import request_handler
|
|
|
|
_logger = logging.getLogger(__name__)
|
|
|
|
|
|
class ExecException(Exception):
|
|
"""Exception type thrown when process-spawning fails on
|
|
the local host. """
|
|
|
|
def __init__(self, cmd, exitstatus, out):
|
|
self.cmd = cmd
|
|
self.exitstatus = exitstatus
|
|
self.out = out.read()
|
|
def __str__(self):
|
|
return 'Command `{self.cmd}\' exited with {self.exitstatus}:\n{self.out}'.format(self=self)
|
|
|
|
|
|
class HostInfo(object):
|
|
"""Class which provides host information from a Linux-style /proc file system."""
|
|
def __init__(self, ch, uname, machine):
|
|
self.ch = ch
|
|
self.pm = posixpath
|
|
self.uname = uname
|
|
self.machine = machine
|
|
self.envcmd = [ 'env' ]
|
|
|
|
@property
|
|
def _host_info_path(self):
|
|
return self.path_module.join(request_handler.configdir, 'host_info', 'binaries', self.uname, self.machine, 'host_info')
|
|
|
|
def _run_host_info(self):
|
|
hostRes = json.loads(self.ch._exec_pkg_cmdv([self._host_info_path]))
|
|
hostRes['uname'] = self.uname
|
|
return { 'host': { 'name' : self.ch.host }, 'hostRes': hostRes }
|
|
|
|
# def _get_hostInfo(self):
|
|
# meminfo = self.ch.open('/proc/meminfo')
|
|
# cpuinfo = self.ch.open('/proc/cpuinfo')
|
|
# try:
|
|
# return { 'host': { 'name' : self.ch.host },
|
|
# 'hostRes': { 'ram' : int(meminfo.readline().split()[1]) / 1024,
|
|
# 'cores': len([ln for ln in cpuinfo.readlines() if 'processor' in ln]),
|
|
# 'uname': self.uname}}
|
|
# finally:
|
|
# meminfo.close()
|
|
# cpuinfo.close()
|
|
|
|
@property
|
|
def ram(self):
|
|
"""Caching?"""
|
|
with self.ch.open('/proc/meminfo') as meminfo:
|
|
return int(meminfo.readline().split()[1]) / 1024
|
|
|
|
@property
|
|
def cores(self):
|
|
"""x"""
|
|
with self.ch.open('/proc/cpuinfo') as cpuinfo:
|
|
return len([ln for ln in cpuinfo.readlines() if 'processor' in ln])
|
|
|
|
@property
|
|
def installdir(self):
|
|
"""x"""
|
|
return None # Don't know how to get this for remote hosts
|
|
|
|
@property
|
|
def homedir(self):
|
|
"""x"""
|
|
return self.ch.env['HOME']
|
|
|
|
# @property
|
|
# def rep(self):
|
|
# """A Python dict representation of the hostInfoRep structure which can be directly converted to Json."""
|
|
# reply = None
|
|
# # try:
|
|
# # reply = self._run_host_info()
|
|
# # except:
|
|
# # _logger.exception('Running host_info failed. Falling back to system tools:')
|
|
|
|
# reply = self._get_hostInfo()
|
|
|
|
# local = platform.system()
|
|
# remote = self.uname
|
|
# if (local == 'Windows' and (remote == 'CYGWIN' or remote == 'Windows')) or (local != 'Windows' and remote != 'CYGWIN' and remote != 'Windows'):
|
|
# _logger.debug('localhost and remote are similar type, apply local paths')
|
|
# _logger.debug('remote env: '+str(self.ch.env))
|
|
# reply['hostRes']['installdir'] = request_handler.basedir
|
|
# reply['hostRes']['datadir'] = self.pm.join(self.pm.expanduser('~'), 'MySQL_Cluster')
|
|
|
|
# return reply
|
|
|
|
@property
|
|
def path_module(self):
|
|
"""Returns the python path module to use when manipulating path names on this host."""
|
|
return self.pm
|
|
|
|
|
|
class SolarisHostInfo(HostInfo):
|
|
"""Specialization for Solaris which uses prtconf and psrinfo to retrieve host information."""
|
|
|
|
# def _get_hostInfo(self):
|
|
# return { 'host': { 'name' : self.ch.host },
|
|
# 'hostRes': { 'ram' : int(self.ch.exec_blocking(['/usr/sbin/prtconf']).split()[7]),
|
|
# 'cores': len(self.ch.exec_blocking(['/usr/sbin/psrinfo']).split('\n')[0:-1]),
|
|
# 'uname': self.uname}}
|
|
@property
|
|
def ram(self):
|
|
return int(self.ch.exec_blocking(['/usr/sbin/prtconf']).split()[7])
|
|
|
|
@property
|
|
def cores(self):
|
|
return len(self.ch.exec_blocking(['/usr/sbin/psrinfo']).split('\n')[0:-1])
|
|
|
|
class MacHostInfo(HostInfo):
|
|
"""Specialization for MacOS which uses sysctl to retrieve host information."""
|
|
|
|
# def _get_hostInfo(self):
|
|
# sysinfo = self.ch.exec_blocking(['/usr/sbin/sysctl', 'hw.'])
|
|
# ram = [int(filter(str.isdigit, ln.split()[1])) for ln in sysinfo.split('\n') if 'hw.memsize:' in ln][0] / 1024 / 1024
|
|
# cores = [int(filter(str.isdigit, ln.split()[1])) for ln in sysinfo.split('\n') if 'hw.ncpu:' in ln][0]
|
|
# return { 'host': { 'name' : self.ch.host },
|
|
# 'hostRes': { 'ram' : ram, 'cores': cores,
|
|
# 'uname': self.uname}}
|
|
|
|
@property
|
|
def ram(self):
|
|
sysinfo = self.ch.exec_blocking(['/usr/sbin/sysctl', 'hw.'])
|
|
return [int(filter(str.isdigit, ln.split()[1])) for ln in sysinfo.split('\n') if 'hw.memsize:' in ln][0] / 1024 / 1024
|
|
|
|
@property
|
|
def cores(self):
|
|
sysinfo = self.ch.exec_blocking(['/usr/sbin/sysctl', 'hw.'])
|
|
return [int(filter(str.isdigit, ln.split()[1])) for ln in sysinfo.split('\n') if 'hw.ncpu:' in ln][0]
|
|
|
|
class CygwinHostInfo(HostInfo):
|
|
"""Specialization for Windows Cygwin which uses systeminfo and wmic to retrieve host information, but retains posixpath as the path module."""
|
|
|
|
@property
|
|
def _host_info_path(self):
|
|
return self.path_module.join('install','host_info', 'Windows', 'host_info.exe')
|
|
|
|
|
|
# def _get_hostInfo(self):
|
|
# sysinfo = self.ch.exec_blocking(['C:/Windows/system32/systeminfo'])
|
|
# ram = [ int(filter(str.isdigit, ln.split()[3])) for ln in sysinfo.split('\n') if 'Total Physical Memory:' in ln][0]
|
|
# self.logger.debug("ram="+str(ram))
|
|
|
|
# wmic = self.ch.exec_blocking(['C:/Windows/System32/Wbem/wmic', 'CPU', 'GET', '/VALUE'])
|
|
# if isinstance(self.ch, LocalClusterHost):
|
|
# wmic = unicode(wmic, 'utf-16')
|
|
|
|
# cores = sum([ int(ln.split('=')[1])
|
|
# for ln in wmic.split('\n') if 'NumberOfCores' in ln ])
|
|
# self.logger.debug("cores="+str(cores))
|
|
# return { 'host': { 'name' : self.ch.host },
|
|
# 'hostRes': { 'ram' : ram, 'cores': cores, 'uname': self.uname}}
|
|
|
|
@property
|
|
def ram(self):
|
|
sysinfo = self.ch.exec_blocking(['C:/Windows/system32/systeminfo'])
|
|
return [ int(filter(str.isdigit, ln.split()[3])) for ln in sysinfo.split('\n') if 'Total Physical Memory:' in ln][0]
|
|
|
|
@property
|
|
def cores(self):
|
|
wmic = self.ch.exec_blocking(['C:/Windows/System32/Wbem/wmic', 'CPU', 'GET', '/VALUE'])
|
|
if isinstance(self.ch, LocalClusterHost):
|
|
wmic = unicode(wmic, 'utf-16')
|
|
|
|
return sum([ int(ln.split('=')[1]) for ln in wmic.split('\n') if 'NumberOfCores' in ln ])
|
|
|
|
class WindowsHostInfo(CygwinHostInfo):
|
|
"""Specialization of CygwinHostInfo for native Windows which uses ntpath as the path module."""
|
|
|
|
def __init__(self, ch, uname, machine):
|
|
super(type(self), self).__init__(ch, uname, machine)
|
|
self.pm = ntpath
|
|
self.envcmd = [ 'cmd.exe', '/c', 'set' ]
|
|
|
|
@property
|
|
def cores(self):
|
|
return int(self.ch.env['NUMBER_OF_PROCESSORS'])
|
|
|
|
@property
|
|
def homedir(self):
|
|
env = self.ch.env
|
|
if env.has_key('USERPROFILE'):
|
|
return env['USERPROFILE']
|
|
|
|
return env['HOMEDRIVE']+env['HOMEPATH']
|
|
|
|
|
|
@property
|
|
def path_module(self):
|
|
return ntpath
|
|
|
|
# Map from uname string to HostInfo type
|
|
hostInfo_map = { 'SunOS' : SolarisHostInfo,
|
|
'Darwin' : MacHostInfo,
|
|
'Windows' : WindowsHostInfo,
|
|
'CYGWIN' : CygwinHostInfo }
|
|
|
|
class ABClusterHost(object):
|
|
"""Base class providing common interface."""
|
|
__meta_class__ = abc.ABCMeta
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
self.drop()
|
|
return False
|
|
|
|
def __init__(self):
|
|
self._hostInfo = None
|
|
self._env = None
|
|
|
|
@abc.abstractmethod
|
|
def _get_system_tuple(self):
|
|
"""Return a string identifying the hosts operating system."""
|
|
pass
|
|
|
|
@abc.abstractmethod
|
|
def _exec_pkg_cmdv(self, cmdv):
|
|
"""Execute a package binary on this ClusterHost."""
|
|
pass
|
|
|
|
|
|
@abc.abstractmethod
|
|
def open(self, filename, mode='r'):
|
|
pass
|
|
|
|
|
|
|
|
@property
|
|
def hostInfo(self):
|
|
"""Create an appropriate HostInfo object for localhost.
|
|
Uses Pythons platform.system() to determine the type of HostInfo object to return.
|
|
"""
|
|
if self._hostInfo == None:
|
|
(system, machine) = self._get_system_tuple()
|
|
if hostInfo_map.has_key(system):
|
|
self._hostInfo = hostInfo_map[system](self, system, machine)
|
|
else:
|
|
_logger.debug('Using default HostInfo for host='+self.host+'('+system+','+machine+')')
|
|
return HostInfo(self, system, machine)
|
|
|
|
return self._hostInfo
|
|
|
|
@property
|
|
def path_module(self):
|
|
"""Path module to use when manipulating file paths."""
|
|
return self.hostInfo.path_module
|
|
|
|
@property
|
|
def env(self):
|
|
if not self._env:
|
|
ctx = { 'str': self.exec_blocking(self.hostInfo.envcmd), 'properties': {} }
|
|
util.parse_properties(ctx)
|
|
self._env = ctx['properties']
|
|
|
|
return self._env
|
|
|
|
@property
|
|
def ram(self):
|
|
return self.hostInfo.ram
|
|
|
|
@property
|
|
def cores(self):
|
|
return self.hostInfo.cores
|
|
|
|
@property
|
|
def uname(self):
|
|
return self.hostInfo.uname
|
|
|
|
@property
|
|
def installdir(self):
|
|
return self.hostInfo.installdir
|
|
|
|
@property
|
|
def homedir(self):
|
|
return self.hostInfo.homedir
|
|
|
|
|
|
@abc.abstractmethod
|
|
def drop(self, paths=[]):
|
|
"""Close open connections and remove files.
|
|
paths - list of files to remove from host before closing connection
|
|
"""
|
|
map(self.rm_r, paths)
|
|
|
|
@abc.abstractmethod
|
|
def file_exists(self, path):
|
|
"""Test for the existence of a file. If the file actually exists,
|
|
its stat object is returned, otherwise None.
|
|
path - file to check the existence of
|
|
"""
|
|
pass
|
|
|
|
@abc.abstractmethod
|
|
def list_dir(self, path):
|
|
"""List the files in a directory.
|
|
path - directory to list
|
|
"""
|
|
pass
|
|
|
|
@abc.abstractmethod
|
|
def mkdir_p(self, path):
|
|
"""Provides mkdir -p type functionality. That is,
|
|
all missing parent directories are also created. If the directory we are trying to
|
|
create already exists, we silently do nothing. If path or any of its parents is not
|
|
a directory an exception is raised.
|
|
path - directory to create on remote host
|
|
"""
|
|
pass
|
|
|
|
@abc.abstractmethod
|
|
def rm_r(self, path):
|
|
"""Provides rm -r type functionality. That is, all files and
|
|
directories are removed recursively.
|
|
path - file or directory to remove
|
|
"""
|
|
pass
|
|
|
|
def auto_complete(self, basedir, locations, executable):
|
|
"""Find the absolute path of an executable given a prefix directory, a set
|
|
of possible directories, and the basename of the executable.
|
|
basedir - basedir of a cluster installation
|
|
locations - list of directories to try
|
|
executable - basename of executable to auto-complete
|
|
"""
|
|
for l in locations:
|
|
choice = posixpath.join(basedir,l,executable)
|
|
if self.file_exists(choice):
|
|
return choice
|
|
raise Exception('Cannot locate '+executable+' in '+posixpath.join(basedir, str(locations))+' on host '+self.host)
|
|
|
|
@abc.abstractmethod
|
|
def _exec_cmdv(self, cmdv, procCtrl, stdinFile):
|
|
pass
|
|
|
|
def exec_cmdv(self, cmdv, procCtrl={ 'waitForCompletion': True },
|
|
stdinFile=None):
|
|
"""Forwards to virtual."""
|
|
return self._exec_cmdv(cmdv, procCtrl, stdinFile)
|
|
|
|
def exec_blocking(self, cmdv):
|
|
"""Convenience method."""
|
|
assert(isinstance(cmdv, list))
|
|
return self.exec_cmdv(cmdv, { 'waitForCompletion': True })
|
|
|
|
def exec_cluster_daemon(self, cdaemonv, waitsec):
|
|
"""Convenience method."""
|
|
assert(isinstance(cdaemonv, list))
|
|
return self.exec_cmdv(cdaemonv, { 'daemonWait': waitsec })
|
|
|
|
|
|
|
|
class LocalClusterHost(ABClusterHost):
|
|
"""Implement the ABClusterHost interface for access to the local host without
|
|
using SSH over Paramiko. Note that this implies that there will be no authentication """
|
|
|
|
def __init__(self, host):
|
|
super(type(self), self).__init__()
|
|
self.host = host
|
|
|
|
def _get_system_tuple(self):
|
|
system = platform.system()
|
|
if system == 'Windows':
|
|
try:
|
|
subprocess.check_call(['uname'])
|
|
except:
|
|
_logger.debug('No uname available, assuming native Windows')
|
|
return (system, platform.uname()[-2])
|
|
else:
|
|
return ('CYGWIN', 'Unknown')
|
|
return (system, platform.uname()[-1])
|
|
|
|
def _exec_pkg_cmdv(self, cmdv):
|
|
"""Locally this just forwards to exec_cmdv."""
|
|
return self.exec_cmdv(cmdv)
|
|
|
|
@property
|
|
def env(self):
|
|
return os.environ
|
|
|
|
@property
|
|
def cores(self):
|
|
return self.hostInfo.cores
|
|
|
|
@property
|
|
def installdir(self):
|
|
return request_handler.basedir
|
|
|
|
def open(self, filename, mode='r'):
|
|
"""Open a file on ABClusterHost."""
|
|
return open(filename, mode)
|
|
|
|
def drop(self, paths=[]):
|
|
"""Close open connections and remove files.
|
|
paths - list of files to remove from host before closing connection
|
|
"""
|
|
map(self.rm_r, paths)
|
|
|
|
def file_exists(self, path):
|
|
"""Test for the existence of a file on the local host. If the file actually exists,
|
|
its stat result object is returned, otherwise None.
|
|
path - file to check the existence of
|
|
"""
|
|
if os.path.exists(path):
|
|
return os.stat(path)
|
|
else:
|
|
return None
|
|
|
|
def list_dir(self, path):
|
|
"""List the files in a directory on the local host. Forwards to os.listdir().
|
|
path - directory to list
|
|
"""
|
|
return os.listdir(path)
|
|
|
|
def mkdir_p(self, path):
|
|
"""Provides mkdir -p type functionality on the local host. Does nothing if
|
|
the directory already exists, otherwise forwards to os.makedirs
|
|
path - directory to create on remote host
|
|
"""
|
|
if os.path.exists(path) and os.path.isdir(path):
|
|
return
|
|
os.makedirs(path)
|
|
|
|
def rm_r(self, path):
|
|
"""Provides rm -r type functionality on the local host. Forwards to os.rmdirs.
|
|
path - file or directory to remove
|
|
"""
|
|
shutil.rmtree(path)
|
|
|
|
def _exec_cmdv(self, cmdv, procCtrl, stdinFile):
|
|
"""Execute an OS command on the local host, using subprocess module.
|
|
cmdv - complete command vector (argv) of the OS command
|
|
procCtrl - procCtrl object from message which controls how the process
|
|
is started (blocking vs non-blocking and output reporting)
|
|
"""
|
|
|
|
# Add nohup when running locally to prevent server from
|
|
# waiting on the children
|
|
if util.get_val(procCtrl, 'nohup'):
|
|
cmdv[:0] = ['nohup']
|
|
output = tempfile.TemporaryFile()
|
|
stdin = output
|
|
if (stdinFile != None):
|
|
stdin = self.open(stdinFile)
|
|
|
|
try:
|
|
if util.get_val(procCtrl, 'waitForCompletion'):
|
|
try:
|
|
subprocess.check_call(cmdv, stdin=stdin, stdout=output, stderr=output)
|
|
except subprocess.CalledProcessError as cpe:
|
|
if cpe.returncode != util.get_val(procCtrl, 'noRaise', 0):
|
|
output.seek(0)
|
|
_logger.error('output='+output.read())
|
|
raise
|
|
output.seek(0)
|
|
return output.read()
|
|
|
|
proc = subprocess.Popen(cmdv, stdin=stdin, stderr=output)
|
|
if proc.poll() == None and procCtrl.has_key('daemonWait'):
|
|
_logger.debug('Popen waits {0} sec for {1}'.format(procCtrl['daemonWait'], ' '.join(cmdv)))
|
|
time.sleep(procCtrl['daemonWait'])
|
|
if proc.poll() != None:
|
|
output.seek(0)
|
|
raise ExecException(' '.join(cmdv), proc.returncode, output)
|
|
finally:
|
|
#output.seek(0)
|
|
#print output.read()
|
|
if (stdin != output):
|
|
stdin.close()
|
|
output.close()
|
|
|
|
def execute_command(self, cmdv, inFile=None):
|
|
"""Execute an OS command blocking on the local host, using
|
|
subprocess module. Returns dict contaning output from process.
|
|
cmdv - complete command vector (argv) of the OS command.
|
|
"""
|
|
outFile = tempfile.TemporaryFile()
|
|
errFile = tempfile.TemporaryFile()
|
|
result = {
|
|
'exitstatus': subprocess.call(args=cmdv, stdin=inFile,
|
|
stdout=outFile, stderr=errFile)
|
|
}
|
|
outFile.seek(0)
|
|
errFile.seek(0)
|
|
result['out'] = outFile.read()
|
|
result['err'] = errFile.read()
|
|
return result
|
|
|
|
def produce_ABClusterHost(hostname='localhost', user=None, pwd=None):
|
|
"""Factory method which returns RemoteClusterHost or LocalClusterHost depending
|
|
on the value of hostname.."""
|
|
|
|
if hostname == 'localhost' or hostname == '127.0.0.1' or hostname == socket.gethostname():
|
|
return LocalClusterHost(hostname)
|
|
|
|
hostname_fqdn = socket.getfqdn(hostname)
|
|
if hostname_fqdn == socket.getfqdn('localhost') or hostname_fqdn == socket.getfqdn(socket.gethostname()):
|
|
return LocalClusterHost(hostname)
|
|
import remote_clusterhost
|
|
return remote_clusterhost.RemoteClusterHost(hostname, user, pwd)
|