Repository :
http://git.fedorahosted.org/cgit/copr.git
On branch : master
---------------------------------------------------------------
commit b57525c79c24dc09baca31ba31ccf7e4d28251ba
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Mon Apr 13 19:42:07 2015 +0200
[backend] script to clean up erred and forgotten VM's using python-novaclient
---------------------------------------------------------------
backend/backend/helpers.py | 15 +++++-
backend/copr-backend.spec | 6 ++
backend/run/cleanup_vm_nova.py | 124 ++++++++++++++++++++++++++++++++++++++++
3 files changed, 144 insertions(+), 1 deletions(-)
diff --git a/backend/backend/helpers.py b/backend/backend/helpers.py
index ab01b91..382716a 100644
--- a/backend/backend/helpers.py
+++ b/backend/backend/helpers.py
@@ -11,9 +11,13 @@ import optparse
import ConfigParser
import os
import sys
-import datetime
+
import traceback
+from datetime import datetime
+import pytz
+# from dateutil.parser import parse as dt_parse
+
from bunch import Bunch
from redis import StrictRedis
from . import constants
@@ -324,3 +328,12 @@ def create_file_logger(name, filepath, fmt=None):
logger.addHandler(handler)
return logger
+
+
+def utc_now():
+ """
+ :return datetime.datetime: Current utc datetime with specified timezone
+ """
+ u = datetime.utcnow()
+ u = u.replace(tzinfo=pytz.utc)
+ return u
diff --git a/backend/copr-backend.spec b/backend/copr-backend.spec
index 51fde49..9d04321 100644
--- a/backend/copr-backend.spec
+++ b/backend/copr-backend.spec
@@ -46,6 +46,9 @@ BuildRequires: ansible >= 1.2
BuildRequires: python-IPy
BuildRequires: python-paramiko
BuildRequires: python-psutil
+BuildRequires: python-futures
+BuildRequires: python-dateutil
+BuildRequires: pytz
# BuildRequires: wget -- ???
#for doc package
@@ -70,6 +73,9 @@ Requires: python-copr
Requires: python-six
Requires: python-IPy
Requires: python-psutil
+Requires: python-futures
+Requires: python-dateutil
+Requires: pytz
Requires: redis
Requires: logrotate
Requires: fedmsg
diff --git a/backend/run/cleanup_vm_nova.py b/backend/run/cleanup_vm_nova.py
new file mode 100755
index 0000000..e23098c
--- /dev/null
+++ b/backend/run/cleanup_vm_nova.py
@@ -0,0 +1,124 @@
+#!/usr/bin/python
+# coding: utf-8
+
+import os
+import sys
+import time
+import logging
+
+from datetime import datetime
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dateutil.parser import parse as dt_parse
+
+import psutil
+import yaml
+from novaclient.v1_1.client import Client
+
+sys.path.append("/usr/share/copr/")
+
+try:
+ from backend.helpers import utc_now
+except ImportError:
+ # TODO: remove when updated version of copr-backend will be released
+ import pytz
+
+ def utc_now():
+ """
+ :return datetime.datetime: Current utc datetime with specified timezone
+ """
+ u = datetime.utcnow()
+ u = u.replace(tzinfo=pytz.utc)
+ return u
+
+
+logging.getLogger("requests").setLevel(logging.ERROR)
+
+
+nova_cloud_vars_path = os.environ.get("NOVA_CLOUD_VARS",
"/home/copr/provision/nova_cloud_vars.yml")
+
+
+def read_config():
+ with open(nova_cloud_vars_path) as handle:
+ conf = yaml.load(handle.read())
+ return conf
+
+
+def get_client(conf):
+ return Client(username=conf["OS_USERNAME"],
+ api_key=conf["OS_PASSWORD"],
+ project_id=conf["OS_TENANT_NAME"],
+ auth_url=conf["OS_AUTH_URL"],
+ insecure=True)
+
+
+class Cleaner(object):
+ def __init__(self, conf):
+ self.conf = conf
+ self.nt = None
+ self.ps_set = None
+
+ def post_init(self):
+ self.nt = get_client(self.conf)
+ # TODO: use VM management after release
+ self.ps_set = "\n".join(p.name + " ".join(p.cmdline) for p in
psutil.process_iter())
+ # log.debug("ps_set: \n{}".format(self.ps_set))
+
+ @staticmethod
+ def terminate(srv):
+ try:
+ srv.delete()
+ log.info("delete invoked for: {}".format(srv))
+ except Exception as err:
+ log.exception("failed to request VM termination: {}".format(err))
+
+ @staticmethod
+ def old_enough(srv):
+ dt_created = dt_parse(srv.created)
+ delta = (utc_now() - dt_created).total_seconds()
+ # log.debug("Server {} created {} now {}; delta: {}".format(srv,
dt_created, utc_now(), delta))
+ return delta > 60 * 10 # 10 minutes
+
+ def check_one(self, srv_id):
+ srv = self.nt.servers.get(srv_id)
+ log.debug("checking vm: {}".format(srv))
+ srv.get()
+ if srv.status == u"ERROR":
+ log.info("server {} got into the error state,
deleting".format(srv))
+ self.terminate(srv)
+ elif self.old_enough(srv) and srv.human_id not in self.ps_set:
+ log.info("server {} not used by any builder".format(srv))
+ self.terminate(srv)
+ # elif not self.old_enough(srv):
+ # log.info("Server {} not old enough".format(srv))
+
+ def main(self):
+ """
+ Terminate erred VM's and VM's with uptime > 10 minutes and which
doesn't have associated process
+ """
+ self.post_init()
+ start = time.time()
+
+ srv_list = self.nt.servers.list(detailed=False)
+ with ThreadPoolExecutor(max_workers=20) as executor:
+ future_check = {executor.submit(self.check_one, srv.id): srv.id for srv in
srv_list}
+ for future in as_completed(future_check):
+ try:
+ future.result()
+ except Exception as exc:
+ log.exception(exc)
+
+ log.info("cleanup consumed: {} seconds".format(time.time() - start))
+
+if __name__ == "__main__":
+ logging.basicConfig(
+ filename="/var/log/copr/cleanup_vms.log",
+ # filename="/tmp/cleanup_vms.log",
+ # stream=sys.stdout,
+ format='[%(asctime)s][%(thread)s][%(levelname)6s]: %(message)s',
+ level=logging.INFO)
+
+ log = logging.getLogger(__name__)
+ log.info("Logger done")
+
+ cleaner = Cleaner(read_config())
+ cleaner.main()