diff --git a/0002-fix-the-problem-of-continuous-spaces.patch b/0002-fix-the-problem-of-continuous-spaces.patch new file mode 100644 index 0000000..f752a3e --- /dev/null +++ b/0002-fix-the-problem-of-continuous-spaces.patch @@ -0,0 +1,255 @@ +diff --git a/packageship/application/initsystem/data_import.py b/packageship/application/initsystem/data_import.py +index c2169c1..a5846bd 100644 +--- a/packageship/application/initsystem/data_import.py ++++ b/packageship/application/initsystem/data_import.py +@@ -84,8 +84,8 @@ class InitDataBase(): + + if not os.path.exists(self.config_file_path): + raise FileNotFoundError( +- 'system initialization configuration file \ +- does not exist: %s' % self.config_file_path) ++ "system initialization configuration file" ++ "does not exist: %s" % self.config_file_path) + # load yaml configuration file + with open(self.config_file_path, 'r', encoding='utf-8') as file_context: + try: +@@ -93,24 +93,25 @@ class InitDataBase(): + file_context.read(), Loader=yaml.FullLoader) + except yaml.YAMLError as yaml_error: + +- raise ConfigurationException(' '.join("The format of the yaml configuration\ +- file is wrong please check and try again:{0}".format(yaml_error).split())) ++ raise ConfigurationException( ++ "The format of the yaml configuration" ++ "file is wrong please check and try again:{0}".format(yaml_error)) + + if init_database_config is None: + raise ConfigurationException( + 'The content of the database initialization configuration file cannot be empty') + if not isinstance(init_database_config, list): + raise ConfigurationException( +- ' '.join('The format of the initial database configuration file\ +- is incorrect.When multiple databases need to be initialized, \ +- it needs to be configured in the form of multiple \ +- nodes:{}'.format(self.config_file_path).split())) ++ "The format of the initial database configuration file" ++ "is incorrect.When multiple databases need to be initialized," ++ "it needs to be configured in the form of multiple" ++ "nodes:{}".format(self.config_file_path)) + for config_item in init_database_config: + if not isinstance(config_item, dict): +- raise ConfigurationException(' '.join('The format of the initial database\ +- configuration file is incorrect, and the value in a single node should\ +- be presented in the form of key - val pairs: \ +- {}'.format(self.config_file_path).split())) ++ raise ConfigurationException( ++ "The format of the initial database" ++ "configuration file is incorrect, and the value in a single node should" ++ "be presented in the form of key - val pairs:{}".format(self.config_file_path)) + return init_database_config + + def init_data(self): +@@ -122,8 +123,8 @@ class InitDataBase(): + """ + if getattr(self, 'config_file_datas', None) is None or \ + self.config_file_datas is None: +- raise ContentNoneException('The content of the database initialization \ +- configuration file is empty') ++ raise ContentNoneException("The content of the database initialization" ++ "configuration file is empty") + + if self.__exists_repeat_database(): + raise DatabaseRepeatException( +@@ -139,13 +140,13 @@ class InitDataBase(): + continue + priority = database_config.get('priority') + if not isinstance(priority, int) or priority < 0 or priority > 100: +- LOGGER.logger.error('The priority value type in the database initialization \ +- configuration file is incorrect') ++ LOGGER.logger.error("The priority value type in the database initialization" ++ "configuration file is incorrect") + continue + lifecycle_status_val = database_config.get('lifecycle') + if lifecycle_status_val not in ('enable', 'disable'): +- LOGGER.logger.error('The status value of the life cycle in the initialization\ +- configuration file can only be enable or disable') ++ LOGGER.logger.error("The value of the life cycle in the initialization" ++ "configuration file can only be enable or disable") + continue + # Initialization data + self._init_data(database_config) +@@ -163,8 +164,8 @@ class InitDataBase(): + """ + _database_engine = self._database_engine.get(self.db_type) + if not _database_engine: +- raise Error('The database engine is set incorrectly, \ +- currently only the following engines are supported: %s ' ++ raise Error("The database engine is set incorrectly," ++ "currently only the following engines are supported: %s " + % '、'.join(self._database_engine.keys())) + _create_table_result = _database_engine( + db_name=db_name, tables=tables, storage=storage).create_database(self) +@@ -200,11 +201,12 @@ class InitDataBase(): + + if src_db_file is None or bin_db_file is None: + raise ContentNoneException( +- 'The path to the sqlite file in the database initialization configuration \ +- is incorrect ') ++ "The path to the sqlite file in the database initialization" ++ "configuration is incorrect ") + if not os.path.exists(src_db_file) or not os.path.exists(bin_db_file): +- raise FileNotFoundError("sqlite file {src} or {bin} does not exist, please \ +- check and try again".format(src=src_db_file, bin=bin_db_file)) ++ raise FileNotFoundError( ++ "sqlite file {src} or {bin} does not exist, please" ++ "check and try again".format(src=src_db_file, bin=bin_db_file)) + # 3. Obtain temporary source package files and binary package files + if self.__save_data(database_config, + self.database_name): +@@ -314,23 +316,20 @@ class InitDataBase(): + + Args: + db_name: Saved database name +- Returns: +- +- Raises: +- + """ + # Query all source packages + self.sql = " select * from packages " + packages_datas = self.__get_data() + if packages_datas is None: + raise ContentNoneException( +- '{db_name}:There is no relevant data in the source \ +- package provided '.format(db_name=db_name)) ++ "{db_name}:There is no relevant data in the source " ++ "package provided ".format(db_name=db_name)) + for index, src_package_item in enumerate(packages_datas): + try: + src_package_name = '-'.join([src_package_item.get('name'), + src_package_item.get('version'), +- src_package_item.get('release') + '.src.rpm' ++ src_package_item.get( ++ 'release') + '.src.rpm' + ]) + except AttributeError as exception_msg: + src_package_name = None +@@ -391,8 +390,9 @@ class InitDataBase(): + self.sql = " select * from requires " + requires_datas = self.__get_data() + if requires_datas is None: +- raise ContentNoneException('{db_name}: The package data that the source package \ +- depends on is empty'.format(db_name=db_name)) ++ raise ContentNoneException( ++ "{db_name}: The package data that the source package " ++ "depends on is empty".format(db_name=db_name)) + with DBHelper(db_name=db_name) as database: + database.batch_add(requires_datas, SrcRequires) + +@@ -411,8 +411,8 @@ class InitDataBase(): + bin_packaegs = self.__get_data() + if bin_packaegs is None: + raise ContentNoneException( +- '{db_name}:There is no relevant data in the provided \ +- binary package '.format(db_name=db_name)) ++ "{db_name}:There is no relevant data in the provided " ++ "binary package ".format(db_name=db_name)) + for index, bin_package_item in enumerate(bin_packaegs): + try: + src_package_name = bin_package_item.get('rpm_sourcerpm').split( +@@ -441,8 +441,8 @@ class InitDataBase(): + requires_datas = self.__get_data() + if requires_datas is None: + raise ContentNoneException( +- '{db_name}:There is no relevant data in the provided binary \ +- dependency package'.format(db_name=db_name)) ++ "{db_name}:There is no relevant data in the provided binary " ++ "dependency package".format(db_name=db_name)) + + with DBHelper(db_name=db_name) as database: + database.batch_add(requires_datas, BinRequires) +@@ -462,8 +462,8 @@ class InitDataBase(): + provides_datas = self.__get_data() + if provides_datas is None: + raise ContentNoneException( +- '{db_name}:There is no relevant data in the provided \ +- binary component '.format(db_name=db_name)) ++ "{db_name}:There is no relevant data in the provided " ++ "binary component ".format(db_name=db_name)) + + with DBHelper(db_name=db_name) as database: + database.batch_add(provides_datas, BinProvides) +@@ -474,8 +474,8 @@ class InitDataBase(): + files_datas = self.__get_data() + if files_datas is None: + raise ContentNoneException( +- '{db_name}:There is no relevant binary file installation\ +- path data in the provided database '.format(db_name=db_name)) ++ "{db_name}:There is no relevant binary file installation " ++ "path data in the provided database ".format(db_name=db_name)) + + with DBHelper(db_name=db_name) as database: + database.batch_add(files_datas, BinFiles) +diff --git a/packageship/libs/dbutils/sqlalchemy_helper.py b/packageship/libs/dbutils/sqlalchemy_helper.py +index a0b22e2..d18b115 100644 +--- a/packageship/libs/dbutils/sqlalchemy_helper.py ++++ b/packageship/libs/dbutils/sqlalchemy_helper.py +@@ -279,8 +279,8 @@ class DBHelper(BaseHelper): + + if not isinstance(dicts, list): + raise TypeError( +- 'The input for bulk insertion must be a dictionary \ +- list with the same fields as the current entity') ++ "The input for bulk insertion must be a dictionary" ++ "list with the same fields as the current entity") + try: + self.session.execute( + model.__table__.insert(), +diff --git a/packageship/pkgship.py b/packageship/pkgship.py +index 884b2ab..f9408c8 100644 +--- a/packageship/pkgship.py ++++ b/packageship/pkgship.py +@@ -25,8 +25,8 @@ try: + + LOGGER = Log(__name__) + except ImportError as import_error: +- print('Error importing related dependencies, \ +- please check if related dependencies are installed') ++ print("Error importing related dependencies," ++ "please check if related dependencies are installed") + else: + from packageship.application.apps.package.function.constants import ResponseCode + from packageship.application.apps.package.function.constants import ListNode +@@ -230,7 +230,9 @@ class PkgshipCommand(BaseCommand): + if package_all.get("not_found_components"): + print("Problem: Not Found Components") + for not_found_com in package_all.get("not_found_components"): +- print(" - nothing provides {} needed by {} ".format(not_found_com, params.packagename)) ++ print( ++ " - nothing provides {} needed by {} ". ++ format(not_found_com, params.packagename)) + package_all = package_all.get("build_dict") + + for bin_package, package_depend in package_all.items(): +@@ -835,7 +837,9 @@ class InstallDepCommand(PkgshipCommand): + if package_all.get("not_found_components"): + print("Problem: Not Found Components") + for not_found_com in package_all.get("not_found_components"): +- print(" - nothing provides {} needed by {} ".format(not_found_com, params.packagename)) ++ print( ++ " - nothing provides {} needed by {} ". ++ format(not_found_com, params.packagename)) + for bin_package, package_depend in package_all.get("install_dict").items(): + # distinguish whether the current data is the data of the root node + if isinstance(package_depend, list) and package_depend[-1][0][0] != 'root': +@@ -1061,7 +1065,9 @@ class SelfBuildCommand(PkgshipCommand): + if package_all.get("not_found_components"): + print("Problem: Not Found Components") + for not_found_com in package_all.get("not_found_components"): +- print(" - nothing provides {} needed by {} ".format(not_found_com, params.packagename)) ++ print( ++ " - nothing provides {} needed by {} ". ++ format(not_found_com, params.packagename)) + bin_package_count = self._parse_bin_package( + package_all.get('binary_dicts')) + diff --git a/0003-fix-log_level-configuration-item-not-work.patch b/0003-fix-log_level-configuration-item-not-work.patch new file mode 100644 index 0000000..3d4b564 --- /dev/null +++ b/0003-fix-log_level-configuration-item-not-work.patch @@ -0,0 +1,55 @@ +diff --git a/packageship/application/__init__.py b/packageship/application/__init__.py +index 1361058..6a57a2e 100644 +--- a/packageship/application/__init__.py ++++ b/packageship/application/__init__.py +@@ -2,8 +2,6 @@ + """ + Initial operation and configuration of the flask project + """ +-import sys +-import threading + from flask import Flask + from flask_session import Session + from flask_apscheduler import APScheduler +@@ -19,7 +17,9 @@ def _timed_task(app): + """ + Timed task function + """ +- from .apps.lifecycle.function.download_yaml import update_pkg_info # pylint: disable=import-outside-toplevel ++ # disable=import-outside-toplevel Avoid circular import problems,so import inside the function ++ # pylint: disable=import-outside-toplevel ++ from packageship.application.apps.lifecycle.function.download_yaml import update_pkg_info + + _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH) + try: +@@ -34,6 +34,7 @@ def _timed_task(app): + if _minute < 0 or _minute > 59: + _minute = 0 + ++ # disable=no-member Dynamic variable pylint is not recognized + app.apscheduler.add_job( # pylint: disable=no-member + func=update_pkg_info, id="update_package_data", trigger="cron", hour=_hour, minute=_minute) + app.apscheduler.add_job( # pylint: disable=no-member +@@ -52,7 +53,8 @@ def init_app(operation): + app = Flask(__name__) + + # log configuration +- app.logger.addHandler(setup_log(Config)) ++ # disable=no-member Dynamic variable pylint is not recognized ++ app.logger.addHandler(setup_log(Config())) # pylint: disable=no-member + + # Load configuration items + +@@ -66,10 +68,12 @@ def init_app(operation): + # Open session function + Session(app) + ++ # Variables OPERATION need to be modified within the function and imported in other modules + global OPERATION # pylint: disable=global-statement + OPERATION = operation + + # Register Blueprint ++ # disable=import-outside-toplevel Avoid circular import problems,so import inside the function + from packageship.application import apps # pylint: disable=import-outside-toplevel + for blue, api in apps.blue_point: + api.init_app(app) diff --git a/0004-fix-the-error-when-executing-query-commands.patch b/0004-fix-the-error-when-executing-query-commands.patch new file mode 100644 index 0000000..04552e5 --- /dev/null +++ b/0004-fix-the-error-when-executing-query-commands.patch @@ -0,0 +1,24 @@ +diff --git a/packageship/application/apps/package/function/packages.py b/packageship/application/apps/package/function/packages.py +index eb96087..d36fc34 100644 +--- a/packageship/application/apps/package/function/packages.py ++++ b/packageship/application/apps/package/function/packages.py +@@ -313,7 +313,8 @@ def _sub_pack(src_name, table_name): + pro_info = res[pro_obj.sub_name]["provides"] + if pro_obj.sub_pro_name in pro_info: + pro_info[pro_obj.sub_pro_name]["requiredby"].update( +- {pro_obj.sub_reqby_name: pro_obj.sub_reqby_name}) ++ {pro_obj.sub_reqby_name: pro_obj.sub_reqby_name} ++ if pro_obj.sub_reqby_name else {}) + else: + pro_info.update( + { +@@ -368,7 +369,8 @@ def _sub_pack(src_name, table_name): + req_info = sub_pkg_info["requires"] + if req_obj.sub_req_name in req_info: + req_info[req_obj.sub_req_name]["providedby"].update( +- {req_obj.sub_proby_name: req_obj.sub_proby_name}) ++ {req_obj.sub_proby_name: req_obj.sub_proby_name} ++ if req_obj.sub_proby_name else {}) + else: + req_info.update( + { diff --git a/0005-fix-the-error-when-source-package-has-no-sub-packages.patch b/0005-fix-the-error-when-source-package-has-no-sub-packages.patch new file mode 100644 index 0000000..6188dbf --- /dev/null +++ b/0005-fix-the-error-when-source-package-has-no-sub-packages.patch @@ -0,0 +1,62 @@ +diff --git a/packageship/application/apps/package/function/self_depend.py b/packageship/application/apps/package/function/self_depend.py +index 1ec4c28..b06b950 100644 +--- a/packageship/application/apps/package/function/self_depend.py ++++ b/packageship/application/apps/package/function/self_depend.py +@@ -106,16 +106,20 @@ class SelfDepend(): + """ + if packtype == 'source': + response_code, subpack_list = self.search_db.get_sub_pack([packname]) +- if subpack_list: +- for subpack_tuple, dbname in subpack_list: +- self.source_dicts.append_src(packname, dbname, subpack_tuple.search_version) +- if dbname != 'NOT FOUND': +- self.binary_dict.append_bin(key=subpack_tuple.subpack_name, +- src=packname, +- version=subpack_tuple.search_version, +- dbname=dbname) +- else: +- return ResponseCode.PACK_NAME_NOT_FOUND ++ if not subpack_list: ++ return ResponseCode.PACK_NAME_NOT_FOUND ++ ++ for subpack_tuple, dbname in subpack_list: ++ self.source_dicts.append_src(packname, dbname, subpack_tuple.search_version) ++ if dbname == 'NOT FOUND': ++ continue ++ ++ if subpack_tuple.subpack_name and subpack_tuple.subpack_name \ ++ not in self.binary_dict.dictionary: ++ self.binary_dict.append_bin(key=subpack_tuple.subpack_name, ++ src=packname, ++ version=subpack_tuple.search_version, ++ dbname=dbname) + + else: + response_code, dbname, source_name, version = \ +@@ -178,15 +182,17 @@ class SelfDepend(): + self.search_subpack_list.remove(None) + _, result_list = self.search_db.get_sub_pack(self.search_subpack_list) + for subpack_tuple, dbname in result_list: +- if dbname != 'NOT FOUND': +- if subpack_tuple.subpack_name and subpack_tuple.subpack_name \ +- not in self.binary_dict.dictionary: +- self.binary_dict.append_bin(key=subpack_tuple.subpack_name, +- src=subpack_tuple.search_name, +- version=subpack_tuple.sub_pack_version, +- dbname=dbname, +- parent_node=[subpack_tuple.search_name, 'Subpack']) +- self.search_install_list.append(subpack_tuple.subpack_name) ++ if dbname == 'NOT FOUND': ++ continue ++ ++ if subpack_tuple.subpack_name and subpack_tuple.subpack_name \ ++ not in self.binary_dict.dictionary: ++ self.binary_dict.append_bin(key=subpack_tuple.subpack_name, ++ src=subpack_tuple.search_name, ++ version=subpack_tuple.sub_pack_version, ++ dbname=dbname, ++ parent_node=[subpack_tuple.search_name, 'Subpack']) ++ self.search_install_list.append(subpack_tuple.subpack_name) + self.search_subpack_list.clear() + + def query_build(self, selfbuild): diff --git a/0006-fix-data-duplication-issue.patch b/0006-fix-data-duplication-issue.patch new file mode 100644 index 0000000..11713f2 --- /dev/null +++ b/0006-fix-data-duplication-issue.patch @@ -0,0 +1,3070 @@ +diff -Naru a/packageship/application/apps/lifecycle/function/concurrent.py b/packageship/application/apps/lifecycle/function/concurrent.py +--- a/packageship/application/apps/lifecycle/function/concurrent.py 2020-09-22 23:34:04.037937224 +0800 ++++ b/packageship/application/apps/lifecycle/function/concurrent.py 2020-09-22 23:48:39.938515522 +0800 +@@ -1,65 +1,80 @@ +-#!/usr/bin/python3 +-""" +-Use queues to implement the producer and consumer model +-to solve the database lock introduced by high concurrency issues +-""" +-import threading +-from queue import Queue +-from sqlalchemy.exc import SQLAlchemyError +-from packageship.libs.dbutils import DBHelper +-from packageship.libs.exception import Error, ContentNoneException +-from packageship.libs.log import Log +- +- +-class ProducerConsumer(): +- """ +- The data written in the database is added to the high +- concurrency queue, and the high concurrency is solved +- by the form of the queue +- """ +- _queue = Queue(maxsize=0) +- _instance_lock = threading.Lock() +- _log = Log(__name__) +- +- def __init__(self): +- self.thread_queue = threading.Thread(target=self.__queue_process) +- if not self.thread_queue.isAlive(): +- self.thread_queue.start() +- +- def start_thread(self): +- """ +- Judge a thread, if the thread is terminated, restart +- """ +- if not self.thread_queue.isAlive(): +- self.thread_queue = threading.Thread(target=self.__queue_process) +- self.thread_queue.start() +- +- def __new__(cls, *args, **kwargs): # pylint: disable=unused-argument +- """ +- Use the singleton pattern to create a thread-safe producer pattern +- """ +- if not hasattr(cls, "_instance"): +- with cls._instance_lock: +- if not hasattr(cls, "_instance"): +- cls._instance = object.__new__(cls) +- return cls._instance +- +- def __queue_process(self): +- """ +- Read the content in the queue and save and update +- """ +- while not self._queue.empty(): +- _queue_value = self._queue.get() +- try: +- with DBHelper(db_name="lifecycle") as database: +- database.add(_queue_value) +- except (Error, ContentNoneException, SQLAlchemyError) as error: +- self._log.logger.error(error) +- +- def put(self, pending_content): +- """ +- The content of the operation is added to the queue +- """ +- if pending_content: +- self._queue.put(pending_content) +- self.start_thread() ++#!/usr/bin/python3 ++""" ++Use queues to implement the producer and consumer model ++to solve the database lock introduced by high concurrency issues ++""" ++import threading ++import time ++from queue import Queue ++from sqlalchemy.exc import SQLAlchemyError ++from sqlalchemy.exc import OperationalError ++from packageship.libs.exception import Error, ContentNoneException ++from packageship.libs.log import Log ++from packageship.libs.configutils.readconfig import ReadConfig ++from packageship import system_config ++ ++ ++class ProducerConsumer(): ++ """ ++ The data written in the database is added to the high ++ concurrency queue, and the high concurrency is solved ++ by the form of the queue ++ """ ++ _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH) ++ queue_maxsize = int(_readconfig.get_config('LIFECYCLE', 'queue_maxsize')) ++ if not isinstance(queue_maxsize, int): ++ queue_maxsize = 1000 ++ _queue = Queue(maxsize=queue_maxsize) ++ _instance_lock = threading.Lock() ++ _log = Log(__name__) ++ ++ def __init__(self): ++ self.thread_queue = threading.Thread(target=self.__queue_process) ++ self._instance_lock.acquire() ++ if not self.thread_queue.isAlive(): ++ self.thread_queue = threading.Thread(target=self.__queue_process) ++ self.thread_queue.start() ++ self._instance_lock.release() ++ ++ def start_thread(self): ++ """ ++ Judge a thread, if the thread is terminated, restart ++ """ ++ self._instance_lock.acquire() ++ if not self.thread_queue.isAlive(): ++ self.thread_queue = threading.Thread(target=self.__queue_process) ++ self.thread_queue.start() ++ self._instance_lock.release() ++ ++ def __new__(cls, *args, **kwargs): # pylint: disable=unused-argument ++ """ ++ Use the singleton pattern to create a thread-safe producer pattern ++ """ ++ if not hasattr(cls, "_instance"): ++ with cls._instance_lock: ++ if not hasattr(cls, "_instance"): ++ cls._instance = object.__new__(cls) ++ return cls._instance ++ ++ def __queue_process(self): ++ """ ++ Read the content in the queue and save and update ++ """ ++ while not self._queue.empty(): ++ _queue_value, method = self._queue.get() ++ try: ++ method(_queue_value) ++ except OperationalError as error: ++ self._log.logger.warning(error) ++ time.sleep(0.2) ++ self._queue.put((_queue_value, method)) ++ except (Error, ContentNoneException, SQLAlchemyError) as error: ++ self._log.logger.error(error) ++ ++ def put(self, pending_content): ++ """ ++ The content of the operation is added to the queue ++ """ ++ if pending_content: ++ self._queue.put(pending_content) ++ self.start_thread() +diff -Naru a/packageship/application/apps/lifecycle/function/download_yaml.py b/packageship/application/apps/lifecycle/function/download_yaml.py +--- a/packageship/application/apps/lifecycle/function/download_yaml.py 2020-09-22 23:34:04.037937224 +0800 ++++ b/packageship/application/apps/lifecycle/function/download_yaml.py 2020-09-22 23:48:46.478549707 +0800 +@@ -1,222 +1,224 @@ +-#!/usr/bin/python3 +-""" +-Dynamically obtain the content of the yaml file \ +-that saves the package information, periodically \ +-obtain the content and save it in the database +-""" +-import copy +-from concurrent.futures import ThreadPoolExecutor +-import datetime as date +-import requests +-import yaml +-from retrying import retry +-from sqlalchemy.exc import SQLAlchemyError +-from requests.exceptions import HTTPError +-from packageship import system_config +-from packageship.application.models.package import Packages +-from packageship.application.models.package import PackagesMaintainer +-from packageship.libs.dbutils import DBHelper +-from packageship.libs.exception import Error, ContentNoneException +-from packageship.libs.configutils.readconfig import ReadConfig +-from .base import Base +-from .gitee import Gitee +-from .concurrent import ProducerConsumer +- +- +-class ParseYaml(): +- """ +- Description: Analyze the downloaded remote yaml file, obtain the tags +- and maintainer information in the yaml file, and save the obtained +- relevant information into the database +- +- Attributes: +- base: base class instance +- pkg: Specific package data +- _table_name: The name of the data table to be operated +- openeuler_advisor_url: Get the warehouse address of the yaml file +- _yaml_content: The content of the yaml file +- """ +- +- def __init__(self, pkg_info, base, table_name): +- self.base = base +- self.pkg = pkg_info +- self._table_name = table_name +- self.openeuler_advisor_url = self._path_stitching(pkg_info.name) +- self._yaml_content = None +- self.timed_task_open = self._timed_task_status() +- self.producer_consumer = ProducerConsumer() +- +- def _timed_task_status(self): +- """ +- The open state of information such as the maintainer in the scheduled task +- """ +- _timed_task_status = True +- _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH) +- open_status = _readconfig.get_config('TIMEDTASK', 'open') +- if open_status not in ('True', 'False'): +- self.base.log.logger.error( +- 'Wrong setting of the open state value of the scheduled task') +- if open_status == 'False': +- self.timed_task_open = False +- return _timed_task_status +- +- def _path_stitching(self, pkg_name): +- """ +- The path of the remote service call +- """ +- _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH) +- _remote_url = _readconfig.get_config('LIFECYCLE', 'warehouse_remote') +- if _remote_url is None: +- _remote_url = 'https://gitee.com/openeuler/openEuler-Advisor/raw/master/upstream-info/' +- return _remote_url + '{pkg_name}.yaml'.format(pkg_name=pkg_name) +- +- def update_database(self): +- """ +- For the current package, determine whether the specific yaml file exists, parse +- the data in it and save it in the database if it exists, and record the relevant +- log if it does not exist +- +- """ +- if self._openeuler_advisor_exists_yaml(): +- self._save_to_database() +- else: +- msg = "The yaml information of the [%s] package has not been" \ +- "obtained yet" % self.pkg.name +- self.base.log.logger.warning(msg) +- +- def _get_yaml_content(self, url): +- """ +- +- """ +- try: +- response = requests.get( +- url, headers=self.base.headers) +- if response.status_code == 200: +- self._yaml_content = yaml.safe_load(response.content) +- +- except HTTPError as error: +- self.base.log.logger.error(error) +- +- def _openeuler_advisor_exists_yaml(self): +- """ +- Determine whether there is a yaml file with the current \ +- package name under the openeuler-advisor project +- +- """ +- self._get_yaml_content(self.openeuler_advisor_url) +- if self._yaml_content: +- return True +- return False +- +- def _save_to_database(self): +- """ +- Save the acquired yaml file information to the database +- +- Raises: +- ContentNoneException: The added entity content is empty +- Error: An error occurred during data addition +- """ +- self._parse_warehouse_info() +- tags = self._yaml_content.get('git_tag', None) +- if tags: +- self._parse_tags_content(tags) +- self.producer_consumer.put(copy.deepcopy(self.pkg)) +- if self.timed_task_open: +- _maintainer = self._yaml_content.get('maintainers') +- if _maintainer and isinstance(_maintainer, list): +- self.pkg.maintainer = _maintainer[0] +- self.pkg.maintainlevel = self._yaml_content.get('maintainlevel') +- try: +- if self.timed_task_open: +- @retry(stop_max_attempt_number=3, stop_max_delay=500) +- def _save_maintainer_info(): +- with DBHelper(db_name="lifecycle") as database: +- _packages_maintainer = database.session.query( +- PackagesMaintainer).filter( +- PackagesMaintainer.name == self.pkg.name).first() +- if _packages_maintainer: +- _packages_maintainer.name = self.pkg.name +- _packages_maintainer.maintainer = self.pkg.maintainer +- _packages_maintainer.maintainlevel = self.pkg.maintainlevel +- else: +- _packages_maintainer = PackagesMaintainer( +- name=self.pkg.name, maintainer=self.pkg.maintainer, +- maintainlevel=self.pkg.maintainlevel) +- self.producer_consumer.put( +- copy.deepcopy(_packages_maintainer)) +- _save_maintainer_info() +- except (Error, ContentNoneException, SQLAlchemyError) as error: +- self.base.log.logger.error(error) +- +- def _parse_warehouse_info(self): +- """ +- Parse the warehouse information in the yaml file +- +- """ +- if self._yaml_content: +- self.pkg.version_control = self._yaml_content.get( +- 'version_control') +- self.pkg.src_repo = self._yaml_content.get('src_repo') +- self.pkg.tag_prefix = self._yaml_content.get('tag_prefix') +- +- def _parse_tags_content(self, tags): +- """ +- Parse the obtained tags content +- +- """ +- try: +- # Integrate tags information into key-value pairs +- _tags = [(tag.split()[0], tag.split()[1]) for tag in tags] +- _tags = sorted(_tags, key=lambda x: x[0], reverse=True) +- self.pkg.latest_version = _tags[0][1] +- self.pkg.latest_version_time = _tags[0][0] +- _end_time = date.datetime.strptime( +- self.pkg.latest_version_time, '%Y-%m-%d') +- if self.pkg.latest_version != self.pkg.version: +- for _version in _tags: +- if _version[1] == self.pkg.version: +- _end_time = date.datetime.strptime( +- _version[0], '%Y-%m-%d') +- self.pkg.used_time = (date.datetime.now() - _end_time).days +- +- except (IndexError, Error) as index_error: +- self.base.log.logger.error(index_error) +- +- +-def update_pkg_info(pkg_info_update=True): +- """ +- Update the information of the upstream warehouse in the source package +- +- """ +- try: +- base_control = Base() +- _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH) +- pool_workers = _readconfig.get_config('LIFECYCLE', 'pool_workers') +- _warehouse = _readconfig.get_config('LIFECYCLE', 'warehouse') +- if _warehouse is None: +- _warehouse = 'src-openeuler' +- if not isinstance(pool_workers, int): +- pool_workers = 10 +- # Open thread pool +- pool = ThreadPoolExecutor(max_workers=pool_workers) +- with DBHelper(db_name="lifecycle") as database: +- for table_name in filter(lambda x: x not in ['packages_issue', 'packages_maintainer'], +- database.engine.table_names()): +- +- cls_model = Packages.package_meta(table_name) +- # Query a specific table +- for package_item in database.session.query(cls_model).all(): +- if pkg_info_update: +- parse_yaml = ParseYaml( +- pkg_info=copy.deepcopy(package_item), +- base=base_control, +- table_name=table_name) +- pool.submit(parse_yaml.update_database) +- else: +- # Get the issue of each warehouse and save it +- gitee_issue = Gitee( +- package_item, _warehouse, package_item.name, table_name) +- pool.submit(gitee_issue.query_issues_info) +- pool.shutdown() +- except SQLAlchemyError as error_msg: +- base_control.log.logger.error(error_msg) ++#!/usr/bin/python3 ++""" ++Dynamically obtain the content of the yaml file \ ++that saves the package information, periodically \ ++obtain the content and save it in the database ++""" ++import copy ++from concurrent.futures import ThreadPoolExecutor ++import datetime as date ++import requests ++import yaml ++from retrying import retry ++from sqlalchemy.exc import SQLAlchemyError ++from requests.exceptions import HTTPError ++from packageship import system_config ++from packageship.application.models.package import Packages ++from packageship.application.models.package import PackagesMaintainer ++from packageship.libs.dbutils import DBHelper ++from packageship.libs.exception import Error, ContentNoneException ++from packageship.libs.configutils.readconfig import ReadConfig ++from .base import Base ++from .gitee import Gitee ++from .concurrent import ProducerConsumer ++ ++ ++class ParseYaml(): ++ """ ++ Description: Analyze the downloaded remote yaml file, obtain the tags ++ and maintainer information in the yaml file, and save the obtained ++ relevant information into the database ++ ++ Attributes: ++ base: base class instance ++ pkg: Specific package data ++ _table_name: The name of the data table to be operated ++ openeuler_advisor_url: Get the warehouse address of the yaml file ++ _yaml_content: The content of the yaml file ++ """ ++ ++ def __init__(self, pkg_info, base, table_name): ++ self.base = base ++ self.pkg = pkg_info ++ self._table_name = table_name ++ self.openeuler_advisor_url = self._path_stitching(pkg_info.name) ++ self._yaml_content = None ++ self.timed_task_open = self._timed_task_status() ++ self.producer_consumer = ProducerConsumer() ++ ++ def _timed_task_status(self): ++ """ ++ The open state of information such as the maintainer in the scheduled task ++ """ ++ _timed_task_status = True ++ _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH) ++ open_status = _readconfig.get_config('TIMEDTASK', 'open') ++ if open_status not in ('True', 'False'): ++ self.base.log.logger.error( ++ 'Wrong setting of the open state value of the scheduled task') ++ if open_status == 'False': ++ self.timed_task_open = False ++ return _timed_task_status ++ ++ def _path_stitching(self, pkg_name): ++ """ ++ The path of the remote service call ++ """ ++ _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH) ++ _remote_url = _readconfig.get_config('LIFECYCLE', 'warehouse_remote') ++ if _remote_url is None: ++ _remote_url = 'https://gitee.com/openeuler/openEuler-Advisor/raw/master/upstream-info/' ++ return _remote_url + '{pkg_name}.yaml'.format(pkg_name=pkg_name) ++ ++ def update_database(self): ++ """ ++ For the current package, determine whether the specific yaml file exists, parse ++ the data in it and save it in the database if it exists, and record the relevant ++ log if it does not exist ++ ++ """ ++ if self._openeuler_advisor_exists_yaml(): ++ self._save_to_database() ++ else: ++ msg = "The yaml information of the [%s] package has not been" \ ++ "obtained yet" % self.pkg.name ++ self.base.log.logger.warning(msg) ++ ++ def _get_yaml_content(self, url): ++ """ ++ ++ """ ++ try: ++ response = requests.get( ++ url, headers=self.base.headers) ++ if response.status_code == 200: ++ self._yaml_content = yaml.safe_load(response.content) ++ ++ except HTTPError as error: ++ self.base.log.logger.error(error) ++ ++ def _openeuler_advisor_exists_yaml(self): ++ """ ++ Determine whether there is a yaml file with the current \ ++ package name under the openeuler-advisor project ++ ++ """ ++ self._get_yaml_content(self.openeuler_advisor_url) ++ if self._yaml_content: ++ return True ++ return False ++ ++ def _save_to_database(self): ++ """ ++ Save the acquired yaml file information to the database ++ ++ Raises: ++ ContentNoneException: The added entity content is empty ++ Error: An error occurred during data addition ++ """ ++ ++ def _save_package(package_module): ++ with DBHelper(db_name="lifecycle") as database: ++ database.add(package_module) ++ ++ def _save_maintainer_info(maintainer_module): ++ with DBHelper(db_name="lifecycle") as database: ++ _packages_maintainer = database.session.query( ++ PackagesMaintainer).filter( ++ PackagesMaintainer.name == maintainer_module['name']).first() ++ if _packages_maintainer: ++ for key, val in maintainer_module.items(): ++ setattr(_packages_maintainer, key, val) ++ else: ++ _packages_maintainer = PackagesMaintainer( ++ **maintainer_module) ++ database.add(_packages_maintainer) ++ ++ self._parse_warehouse_info() ++ tags = self._yaml_content.get('git_tag', None) ++ if tags: ++ self._parse_tags_content(tags) ++ self.producer_consumer.put( ++ (copy.deepcopy(self.pkg), _save_package)) ++ if self.timed_task_open: ++ maintainer = {'name': self.pkg.name} ++ _maintainer = self._yaml_content.get('maintainers') ++ if _maintainer and isinstance(_maintainer, list): ++ maintainer['maintainer'] = _maintainer[0] ++ maintainer['maintainlevel'] = self._yaml_content.get( ++ 'maintainlevel') ++ ++ self.producer_consumer.put((maintainer, _save_maintainer_info)) ++ ++ def _parse_warehouse_info(self): ++ """ ++ Parse the warehouse information in the yaml file ++ ++ """ ++ if self._yaml_content: ++ self.pkg.version_control = self._yaml_content.get( ++ 'version_control') ++ self.pkg.src_repo = self._yaml_content.get('src_repo') ++ self.pkg.tag_prefix = self._yaml_content.get('tag_prefix') ++ ++ def _parse_tags_content(self, tags): ++ """ ++ Parse the obtained tags content ++ ++ """ ++ try: ++ # Integrate tags information into key-value pairs ++ _tags = [(tag.split()[0], tag.split()[1]) for tag in tags] ++ _tags = sorted(_tags, key=lambda x: x[0], reverse=True) ++ self.pkg.latest_version = _tags[0][1] ++ self.pkg.latest_version_time = _tags[0][0] ++ _end_time = date.datetime.strptime( ++ self.pkg.latest_version_time, '%Y-%m-%d') ++ if self.pkg.latest_version != self.pkg.version: ++ for _version in _tags: ++ if _version[1] == self.pkg.version: ++ _end_time = date.datetime.strptime( ++ _version[0], '%Y-%m-%d') ++ self.pkg.used_time = (date.datetime.now() - _end_time).days ++ ++ except (IndexError, Error) as index_error: ++ self.base.log.logger.error(index_error) ++ ++ ++def update_pkg_info(pkg_info_update=True): ++ """ ++ Update the information of the upstream warehouse in the source package ++ ++ """ ++ try: ++ base_control = Base() ++ _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH) ++ pool_workers = _readconfig.get_config('LIFECYCLE', 'pool_workers') ++ _warehouse = _readconfig.get_config('LIFECYCLE', 'warehouse') ++ if _warehouse is None: ++ _warehouse = 'src-openeuler' ++ if not isinstance(pool_workers, int): ++ pool_workers = 10 ++ # Open thread pool ++ pool = ThreadPoolExecutor(max_workers=pool_workers) ++ with DBHelper(db_name="lifecycle") as database: ++ for table_name in filter(lambda x: x not in ['packages_issue', 'packages_maintainer', 'database_info'], ++ database.engine.table_names()): ++ ++ cls_model = Packages.package_meta(table_name) ++ # Query a specific table ++ for package_item in database.session.query(cls_model).all(): ++ if pkg_info_update: ++ parse_yaml = ParseYaml( ++ pkg_info=copy.deepcopy(package_item), ++ base=base_control, ++ table_name=table_name) ++ pool.submit(parse_yaml.update_database) ++ else: ++ # Get the issue of each warehouse and save it ++ gitee_issue = Gitee( ++ copy.deepcopy(package_item), _warehouse, package_item.name, table_name) ++ pool.submit(gitee_issue.query_issues_info) ++ pool.shutdown() ++ except SQLAlchemyError as error_msg: ++ base_control.log.logger.error(error_msg) +diff -Naru a/packageship/application/apps/lifecycle/function/gitee.py b/packageship/application/apps/lifecycle/function/gitee.py +--- a/packageship/application/apps/lifecycle/function/gitee.py 2020-09-22 23:34:04.037937224 +0800 ++++ b/packageship/application/apps/lifecycle/function/gitee.py 2020-09-22 23:48:52.698582219 +0800 +@@ -1,224 +1,223 @@ +-#!/usr/bin/python3 +-""" +-Description:Get issue info from gitee +-Class: Gitee +-""" +-import copy +-from json import JSONDecodeError +-from retrying import retry +-import requests +-from requests.exceptions import HTTPError +-from sqlalchemy.exc import SQLAlchemyError +-from packageship.libs.dbutils import DBHelper +-from packageship.libs.configutils.readconfig import ReadConfig +-from packageship.libs.exception import Error, ContentNoneException +-from packageship.application.models.package import PackagesIssue +-from packageship import system_config +-from packageship.libs.log import Log +-from .concurrent import ProducerConsumer +- +-LOGGER = Log(__name__) +- +- +-class Gitee(): +- """ +- gitee version management tool related information acquisition +- +- """ +- +- def __init__(self, pkg_info, owner, repo, table_name): +- self.pkg_info = pkg_info +- self.owner = owner +- self.repo = repo +- self._read_config = ReadConfig(system_config.SYS_CONFIG_PATH) +- self.url = "https://gitee.com/" +- self.api_url = "https://gitee.com/api/v5/repos" +- self.pool = None +- self.issue_id = None +- self.defect = 0 +- self.feature = 0 +- self.cve = 0 +- self.patch_files_path = self._read_config.get_system( +- "patch_files_path") +- self.table_name = table_name +- self.producer_consumer = ProducerConsumer() +- +- def query_issues_info(self, issue_id=""): +- """ +- Description: View the issue details of the specified package +- Args: +- issue_id: Issue id +- Returns: +- issue_content_list: The issue details of the specified package list +- Raises: +- +- """ +- issue_url = self.api_url + \ +- "/{}/{}/issues/{}".format(self.owner, self.repo, issue_id) +- try: +- response = requests.get( +- issue_url, params={"state": "all", "per_page": 100}) +- except Error as error: +- LOGGER.logger.error(error) +- return None +- if response.status_code != 200: +- return None +- total_page = 1 if issue_id else int(response.headers['total_page']) +- total_count = int(response.headers['total_count']) +- if total_count > 0: +- issue_list = self._query_per_page_issue_info(total_page, issue_url) +- if not issue_list: +- LOGGER.logger.error( +- "An error occurred while querying {}".format(self.repo)) +- return None +- self._save_issues(issue_list) +- +- def _query_per_page_issue_info(self, total_page, issue_url): +- """ +- Description: View the issue details +- Args: +- total_page: total page +- issue_url: issue url +- +- Returns: +- +- """ +- issue_content_list = [] +- for i in range(1, total_page + 1): +- +- @retry(stop_max_attempt_number=3, stop_max_delay=1000) +- def request_issue(page, issue_url): +- try: +- response = requests.get(issue_url, +- params={"state": "all", "per_page": 100, "page": page}) +- except HTTPError: +- raise HTTPError('Network request error') +- return response +- +- try: +- response = request_issue(i, issue_url) +- if response.status_code != 200: +- LOGGER.logger.warning(response.content.decode("utf-8")) +- continue +- issue_content_list.extend( +- self.parse_issues_content(response.json())) +- except (JSONDecodeError, Error) as error: +- LOGGER.logger.error(error) +- return issue_content_list +- +- def _save_issues(self, issue_list): +- """ +- Save the obtained issue information +- +- """ +- try: +- issue_ids = [issue['issue_id'] for issue in issue_list] +- with DBHelper(db_name="lifecycle") as database: +- +- @retry(stop_max_attempt_number=3, stop_max_delay=500) +- def _query_pkgissues(): +- exist_issues = database.session.query(PackagesIssue).filter( +- PackagesIssue.issue_id.in_(issue_ids)).all() # pylint: disable=protected-access +- return exist_issues +- +- exist_issues = _query_pkgissues() +- # Save the issue +- for issue_item in issue_list: +- issue_model = [ +- issue for issue in exist_issues if issue.issue_id == issue_item['issue_id']] +- if issue_model: +- for key, val in issue_item.items(): +- setattr(issue_model[0], key, val) +- self.producer_consumer.put( +- copy.deepcopy(issue_model[0])) +- else: +- self.producer_consumer.put( +- PackagesIssue(**issue_item)) +- +- # The number of various issues in the update package +- self.pkg_info.defect = self.defect +- self.pkg_info.feature = self.feature +- self.pkg_info.cve = self.cve +- self.producer_consumer.put(copy.deepcopy(self.pkg_info)) +- +- except (Error, ContentNoneException, SQLAlchemyError) as error: +- LOGGER.logger.error( +- 'An abnormal error occurred while saving related issues:%s' % error if error else '') +- +- def parse_issues_content(self, sources): +- """ +- Description: Parse the response content and get issue content +- Args:Issue list +- +- Returns:list:issue_id, issue_url, issue_content, issue_status, issue_download +- Raises: +- """ +- result_list = [] +- if isinstance(sources, list): +- for source in sources: +- issue_content = self.parse_issue_content(source) +- if issue_content: +- result_list.append(issue_content) +- else: +- issue_content = self.parse_issue_content(sources) +- if issue_content: +- result_list.append(issue_content) +- return result_list +- +- def parse_issue_content(self, source): +- """ +- Description: Parse the response content and get issue content +- Args: Source of issue content +- +- Returns:list:issue_id, issue_url, issue_content, issue_status, issue_download, issue_status +- issue_type, related_release +- Raises:KeyError +- """ +- try: +- result_dict = {"issue_id": source['number'], "issue_url": source['html_url'], +- "issue_title": source['title'].strip(), +- "issue_content": source['body'].strip(), +- "issue_status": source['state'], "issue_download": "", +- "issue_type": source["issue_type"], +- "pkg_name": self.repo, +- "related_release": source["labels"][0]['name'] if source["labels"] else ''} +- if source["issue_type"] == "缺陷": +- self.defect += 1 +- elif source["issue_type"] == "需求": +- self.feature += 1 +- elif source["issue_type"] == "CVE和安全问题": +- self.cve += 1 +- else: +- pass +- except KeyError as error: +- LOGGER.logger.error(error) +- return None +- return result_dict +- +- def issue_hooks(self, issue_hook_info): +- """ +- Description: Hook data triggered by a new task operation +- Args: +- issue_hook_info: Issue info +- Returns: +- +- Raises: +- +- """ +- if issue_hook_info is None: +- raise ContentNoneException( +- 'The content cannot be empty') +- issue_info_list = [] +- issue_info = issue_hook_info["issue"] +- issue_content = self.parse_issue_content(issue_info) +- if issue_content: +- issue_info_list.append(issue_content) +- if self.feature != 0: +- self.defect, self.feature, self.cve = self.pkg_info.defect, self.pkg_info.feature + \ +- 1, self.pkg_info.cve +- if self.defect != 0: +- self.defect, self.feature, self.cve = self.pkg_info.defect + \ +- 1, self.pkg_info.feature, self.pkg_info.cve +- if self.cve != 0: +- self.defect, self.feature, self.cve = self.pkg_info.defect, self.pkg_info.feature, self.pkg_info.cve + 1 +- self._save_issues(issue_info_list) ++#!/usr/bin/python3 ++""" ++Description:Get issue info from gitee ++Class: Gitee ++""" ++import copy ++from json import JSONDecodeError ++from retrying import retry ++import requests ++from requests.exceptions import HTTPError ++from sqlalchemy.exc import SQLAlchemyError ++from packageship.libs.dbutils import DBHelper ++from packageship.libs.configutils.readconfig import ReadConfig ++from packageship.libs.exception import Error, ContentNoneException ++from packageship.application.models.package import PackagesIssue ++from packageship import system_config ++from packageship.libs.log import Log ++from .concurrent import ProducerConsumer ++ ++LOGGER = Log(__name__) ++ ++ ++class Gitee(): ++ """ ++ gitee version management tool related information acquisition ++ ++ """ ++ ++ def __init__(self, pkg_info, owner, repo, table_name): ++ self.pkg_info = pkg_info ++ self.owner = owner ++ self.repo = repo ++ self._read_config = ReadConfig(system_config.SYS_CONFIG_PATH) ++ self.url = "https://gitee.com/" ++ self.api_url = "https://gitee.com/api/v5/repos" ++ self.pool = None ++ self.issue_id = None ++ self.defect = 0 ++ self.feature = 0 ++ self.cve = 0 ++ self.patch_files_path = self._read_config.get_system( ++ "patch_files_path") ++ self.table_name = table_name ++ self.producer_consumer = ProducerConsumer() ++ ++ def query_issues_info(self, issue_id=""): ++ """ ++ Description: View the issue details of the specified package ++ Args: ++ issue_id: Issue id ++ Returns: ++ issue_content_list: The issue details of the specified package list ++ Raises: ++ ++ """ ++ issue_url = self.api_url + \ ++ "/{}/{}/issues/{}".format(self.owner, self.repo, issue_id) ++ try: ++ response = requests.get( ++ issue_url, params={"state": "all", "per_page": 100}) ++ except Error as error: ++ LOGGER.logger.error(error) ++ return None ++ if response.status_code != 200: ++ return None ++ total_page = 1 if issue_id else int(response.headers['total_page']) ++ total_count = int(response.headers['total_count']) ++ if total_count > 0: ++ issue_list = self._query_per_page_issue_info(total_page, issue_url) ++ if not issue_list: ++ LOGGER.logger.error( ++ "An error occurred while querying {}".format(self.repo)) ++ return None ++ self._save_issues(issue_list) ++ ++ def _query_per_page_issue_info(self, total_page, issue_url): ++ """ ++ Description: View the issue details ++ Args: ++ total_page: total page ++ issue_url: issue url ++ ++ Returns: ++ ++ """ ++ issue_content_list = [] ++ for i in range(1, total_page + 1): ++ ++ @retry(stop_max_attempt_number=3, stop_max_delay=1000) ++ def request_issue(page, issue_url): ++ try: ++ response = requests.get(issue_url, ++ params={"state": "all", "per_page": 100, "page": page}) ++ except HTTPError: ++ raise HTTPError('Network request error') ++ return response ++ ++ try: ++ response = request_issue(i, issue_url) ++ if response.status_code != 200: ++ LOGGER.logger.warning(response.content.decode("utf-8")) ++ continue ++ issue_content_list.extend( ++ self.parse_issues_content(response.json())) ++ except (JSONDecodeError, Error) as error: ++ LOGGER.logger.error(error) ++ return issue_content_list ++ ++ def _save_issues(self, issue_list): ++ """ ++ Save the obtained issue information ++ ++ """ ++ try: ++ def _save(issue_module): ++ with DBHelper(db_name='lifecycle') as database: ++ ++ exist_issues = database.session.query(PackagesIssue).filter( ++ PackagesIssue.issue_id == issue_module['issue_id']).first() ++ if exist_issues: ++ ++ # Save the issue ++ for key, val in issue_module.items(): ++ setattr(exist_issues, key, val) ++ else: ++ exist_issues = PackagesIssue(**issue_module) ++ database.add(exist_issues) ++ ++ def _save_package(package_module): ++ with DBHelper(db_name='lifecycle') as database: ++ database.add(package_module) ++ ++ for issue_item in issue_list: ++ self.producer_consumer.put( ++ (copy.deepcopy(issue_item), _save)) ++ ++ # The number of various issues in the update package ++ self.pkg_info.defect = self.defect ++ self.pkg_info.feature = self.feature ++ self.pkg_info.cve = self.cve ++ self.producer_consumer.put((copy.deepcopy(self.pkg_info), _save_package)) ++ ++ except (Error, ContentNoneException, SQLAlchemyError) as error: ++ LOGGER.logger.error( ++ 'An abnormal error occurred while saving related issues:%s' % error if error else '') ++ ++ def parse_issues_content(self, sources): ++ """ ++ Description: Parse the response content and get issue content ++ Args:Issue list ++ ++ Returns:list:issue_id, issue_url, issue_content, issue_status, issue_download ++ Raises: ++ """ ++ result_list = [] ++ if isinstance(sources, list): ++ for source in sources: ++ issue_content = self.parse_issue_content(source) ++ if issue_content: ++ result_list.append(issue_content) ++ else: ++ issue_content = self.parse_issue_content(sources) ++ if issue_content: ++ result_list.append(issue_content) ++ return result_list ++ ++ def parse_issue_content(self, source): ++ """ ++ Description: Parse the response content and get issue content ++ Args: Source of issue content ++ ++ Returns:list:issue_id, issue_url, issue_content, issue_status, issue_download, issue_status ++ issue_type, related_release ++ Raises:KeyError ++ """ ++ try: ++ result_dict = {"issue_id": source['number'], "issue_url": source['html_url'], ++ "issue_title": source['title'].strip(), ++ "issue_content": source['body'].strip(), ++ "issue_status": source['state'], "issue_download": "", ++ "issue_type": source["issue_type"], ++ "pkg_name": self.repo, ++ "related_release": source["labels"][0]['name'] if source["labels"] else ''} ++ if source["issue_type"] == "缺陷": ++ self.defect += 1 ++ elif source["issue_type"] == "需求": ++ self.feature += 1 ++ elif source["issue_type"] == "CVE和安全问题": ++ self.cve += 1 ++ else: ++ pass ++ except KeyError as error: ++ LOGGER.logger.error(error) ++ return None ++ return result_dict ++ ++ def issue_hooks(self, issue_hook_info): ++ """ ++ Description: Hook data triggered by a new task operation ++ Args: ++ issue_hook_info: Issue info ++ Returns: ++ ++ Raises: ++ ++ """ ++ if issue_hook_info is None: ++ raise ContentNoneException( ++ 'The content cannot be empty') ++ issue_info_list = [] ++ issue_info = issue_hook_info["issue"] ++ issue_content = self.parse_issue_content(issue_info) ++ if issue_content: ++ issue_info_list.append(issue_content) ++ if self.feature != 0: ++ self.defect, self.feature, self.cve = self.pkg_info.defect, self.pkg_info.feature + \ ++ 1, self.pkg_info.cve ++ if self.defect != 0: ++ self.defect, self.feature, self.cve = self.pkg_info.defect + \ ++ 1, self.pkg_info.feature, self.pkg_info.cve ++ if self.cve != 0: ++ self.defect, self.feature, self.cve = self.pkg_info.defect, self.pkg_info.feature, self.pkg_info.cve + 1 ++ self._save_issues(issue_info_list) +diff -Naru a/packageship/application/apps/lifecycle/view.py b/packageship/application/apps/lifecycle/view.py +--- a/packageship/application/apps/lifecycle/view.py 2020-09-22 23:34:04.037937224 +0800 ++++ b/packageship/application/apps/lifecycle/view.py 2020-09-22 23:52:49.731821183 +0800 +@@ -1,760 +1,760 @@ +-#!/usr/bin/python3 +-""" +-Life cycle related api interface +-""" +-import io +-import json +-import math +-import os +-from concurrent.futures import ThreadPoolExecutor +- +-import pandas as pd +-import yaml +- +-from flask import request +-from flask import jsonify, make_response +-from flask import current_app +-from flask_restful import Resource +-from marshmallow import ValidationError +- +-from sqlalchemy.exc import DisconnectionError, SQLAlchemyError +- +-from packageship import system_config +-from packageship.libs.configutils.readconfig import ReadConfig +-from packageship.libs.exception import Error +-from packageship.application.apps.package.function.constants import ResponseCode +-from packageship.libs.dbutils.sqlalchemy_helper import DBHelper +-from packageship.application.models.package import PackagesIssue +-from packageship.application.models.package import Packages +-from packageship.application.models.package import PackagesMaintainer +-from packageship.libs.log import Log +-from .serialize import IssueDownloadSchema, PackagesDownloadSchema, IssuePageSchema, IssueSchema +-from ..package.serialize import DataFormatVerfi, UpdatePackagesSchema +-from .function.gitee import Gitee as gitee +- +-LOGGER = Log(__name__) +- +- +-# pylint: disable = no-self-use +- +-class DownloadFile(Resource): +- """ +- Download the content of the issue or the excel file of the package content +- """ +- +- def _download_excel(self, file_type, table_name=None): +- """ +- Download excel file +- """ +- file_name = 'packages.xlsx' +- if file_type == 'packages': +- download_content = self.__get_packages_content(table_name) +- else: +- file_name = 'issues.xlsx' +- download_content = self.__get_issues_content() +- if download_content is None: +- return jsonify( +- ResponseCode.response_json( +- ResponseCode.SERVICE_ERROR)) +- pd_dataframe = self.__to_dataframe(download_content) +- +- _response = self.__bytes_save(pd_dataframe) +- return self.__set_response_header(_response, file_name) +- +- def __bytes_save(self, data_frame): +- """ +- Save the file content in the form of a binary file stream +- """ +- try: +- bytes_io = io.BytesIO() +- writer = pd.ExcelWriter( # pylint: disable=abstract-class-instantiated +- bytes_io, engine='xlsxwriter') +- data_frame.to_excel(writer, sheet_name='Summary', index=False) +- writer.save() +- writer.close() +- bytes_io.seek(0) +- _response = make_response(bytes_io.getvalue()) +- bytes_io.close() +- return _response +- except (IOError, Error) as io_error: +- current_app.logger.error(io_error) +- return make_response() +- +- def __set_response_header(self, response, file_name): +- """ +- Set http response header information +- """ +- response.headers['Content-Type'] = \ +- "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" +- response.headers["Cache-Control"] = "no-cache" +- response.headers['Content-Disposition'] = 'attachment; filename={file_name}'.format( +- file_name=file_name) +- return response +- +- def __get_packages_content(self, table_name): +- """ +- Get package list information +- """ +- try: +- with DBHelper(db_name='lifecycle') as database: +- # Query all package data in the specified table +- _model = Packages.package_meta(table_name) +- _packageinfos = database.session.query(_model).all() +- packages_dicts = PackagesDownloadSchema( +- many=True).dump(_packageinfos) +- return packages_dicts +- +- except (SQLAlchemyError, DisconnectionError) as error: +- current_app.logger.error(error) +- return None +- +- def __get_issues_content(self): +- """ +- Get the list of issues +- """ +- try: +- with DBHelper(db_name='lifecycle') as database: +- _issues = database.session.query(PackagesIssue).all() +- issues_dicts = IssueDownloadSchema(many=True).dump(_issues) +- return issues_dicts +- except (SQLAlchemyError, DisconnectionError) as error: +- current_app.logger.error(error) +- return None +- +- def __to_dataframe(self, datas): +- """ +- Convert the obtained information into pandas content format +- """ +- +- data_frame = pd.DataFrame(datas) +- return data_frame +- +- def get(self, file_type): +- """ +- Download package collection information and isse list information +- +- """ +- if file_type not in ['packages', 'issues']: +- return jsonify( +- ResponseCode.response_json( +- ResponseCode.PARAM_ERROR)) +- +- table_name = request.args.get('table_name', None) +- response = self._download_excel(file_type, table_name) +- return response +- +- +-class MaintainerView(Resource): +- """ +- Maintainer name collection +- """ +- +- def __query_maintainers(self): +- """ +- Query the names of all maintainers in the specified table +- """ +- try: +- with DBHelper(db_name='lifecycle') as database: +- maintainers = database.session.query( +- PackagesMaintainer.maintainer).group_by(PackagesMaintainer.maintainer).all() +- return [maintainer_item[0] for maintainer_item in maintainers +- if maintainer_item[0]] +- except (SQLAlchemyError, DisconnectionError) as error: +- current_app.logger.error(error) +- return [] +- +- def get(self): +- """ +- Get the list of maintainers +- """ +- # Group query of the names of all maintainers in the current table +- maintainers = self.__query_maintainers() +- return jsonify(ResponseCode.response_json( +- ResponseCode.SUCCESS, +- maintainers)) +- +- +-class TableColView(Resource): +- """ +- The default column of the package shows the interface +- """ +- +- def __columns_names(self): +- """ +- Mapping of column name and title +- """ +- columns = [ +- ('name', 'Name', True), +- ('version', 'Version', True), +- ('release', 'Release', True), +- ('url', 'Url', True), +- ('rpm_license', 'License', False), +- ('feature', 'Feature', False), +- ('maintainer', 'Maintainer', True), +- ('maintainlevel', 'Maintenance Level', True), +- ('release_time', 'Release Time', False), +- ('used_time', 'Used Time', True), +- ('maintainer_status', 'Maintain Status', True), +- ('latest_version', 'Latest Version', False), +- ('latest_version_time', 'Latest Version Release Time', False), +- ('issue', 'Issue', True)] +- return columns +- +- def __columns_mapping(self): +- """ +- +- """ +- columns = list() +- for column in self.__columns_names(): +- columns.append({ +- 'column_name': column[0], +- 'label': column[1], +- 'default_selected': column[2] +- }) +- return columns +- +- def get(self): +- """ +- Get the default display column of the package +- +- """ +- table_mapping_columns = self.__columns_mapping() +- return jsonify( +- ResponseCode.response_json( +- ResponseCode.SUCCESS, +- table_mapping_columns)) +- +- +-class LifeTables(Resource): +- """ +- description: LifeTables +- Restful API: get +- ChangeLog: +- """ +- +- def get(self): +- """ +- return all table names in the database +- +- Returns: +- Return the table names in the database as a list +- """ +- try: +- with DBHelper(db_name="lifecycle") as database_name: +- # View all table names in the package-info database +- all_table_names = database_name.engine.table_names() +- all_table_names.remove("packages_issue") +- all_table_names.remove("packages_maintainer") +- return jsonify( +- ResponseCode.response_json( +- ResponseCode.SUCCESS, data=all_table_names) +- ) +- except (SQLAlchemyError, DisconnectionError, Error, ValueError) as sql_error: +- LOGGER.logger.error(sql_error) +- return jsonify( +- ResponseCode.response_json(ResponseCode.DATABASE_NOT_FOUND) +- ) +- +- +-class IssueView(Resource): +- """ +- Issue content collection +- """ +- +- def _query_issues(self, request_data): +- """ +- Args: +- request_data: +- Returns: +- """ +- try: +- with DBHelper(db_name='lifecycle') as database: +- issues_query = database.session.query(PackagesIssue.issue_id, +- PackagesIssue.issue_url, +- PackagesIssue.issue_title, +- PackagesIssue.issue_status, +- PackagesIssue.pkg_name, +- PackagesIssue.issue_type, +- PackagesMaintainer.maintainer). \ +- outerjoin(PackagesMaintainer, +- PackagesMaintainer.name == PackagesIssue.pkg_name) +- if request_data.get("pkg_name"): +- issues_query = issues_query.filter( +- PackagesIssue.pkg_name == request_data.get("pkg_name")) +- if request_data.get("issue_type"): +- issues_query = issues_query.filter( +- PackagesIssue.issue_type == request_data.get("issue_type")) +- if request_data.get("issue_status"): +- issues_query = issues_query.filter( +- PackagesIssue.issue_status == request_data.get("issue_status")) +- if request_data.get("maintainer"): +- issues_query = issues_query.filter( +- PackagesMaintainer.maintainer == request_data.get("maintainer")) +- total_count = issues_query.count() +- total_page = math.ceil( +- total_count / int(request_data.get("page_size"))) +- issues_query = issues_query.limit(request_data.get("page_size")).offset( +- (int(request_data.get("page_num")) - 1) * int(request_data.get("page_size"))) +- issue_dicts = IssuePageSchema( +- many=True).dump(issues_query.all()) +- issue_data = ResponseCode.response_json( +- ResponseCode.SUCCESS, issue_dicts) +- issue_data['total_count'] = total_count +- issue_data['total_page'] = total_page +- return issue_data +- except (SQLAlchemyError, DisconnectionError) as error: +- current_app.logger.error(error) +- return ResponseCode.response_json(ResponseCode.DATABASE_NOT_FOUND) +- +- def get(self): +- """ +- Description: Get all issues info or one specific issue +- Args: +- Returns: +- [ +- { +- "issue_id": "", +- "issue_url": "", +- "issue_title": "", +- "issue_content": "", +- "issue_status": "", +- "issue_type": "" +- }, +- ] +- Raises: +- DisconnectionError: Unable to connect to database exception +- AttributeError: Object does not have this property +- TypeError: Exception of type +- Error: Abnormal error +- """ +- schema = IssueSchema() +- if schema.validate(request.args): +- return jsonify( +- ResponseCode.response_json(ResponseCode.PARAM_ERROR) +- ) +- issue_dict = self._query_issues(request.args) +- return issue_dict +- +- +-class IssueType(Resource): +- """ +- Issue type collection +- """ +- +- def _get_issue_type(self): +- """ +- Description: Query issue type +- Returns: +- """ +- try: +- with DBHelper(db_name='lifecycle') as database: +- issues_query = database.session.query(PackagesIssue.issue_type).group_by( +- PackagesIssue.issue_type).all() +- return jsonify(ResponseCode.response_json( +- ResponseCode.SUCCESS, [issue_query[0] for issue_query in issues_query])) +- except (SQLAlchemyError, DisconnectionError) as error: +- current_app.logger.error(error) +- return jsonify(ResponseCode.response_json( +- ResponseCode.PARAM_ERROR)) +- +- def get(self): +- """ +- Description: Get all issues info or one specific issue +- Args: +- Returns: +- [ +- "issue_type", +- "issue_type" +- ] +- Raises: +- DisconnectionError: Unable to connect to database exception +- AttributeError: Object does not have this property +- TypeError: Exception of type +- Error: Abnormal error +- """ +- return self._get_issue_type() +- +- +-class IssueStatus(Resource): +- """ +- Issue status collection +- """ +- +- def _get_issue_status(self): +- """ +- Description: Query issue status +- Returns: +- """ +- try: +- with DBHelper(db_name='lifecycle') as database: +- issues_query = database.session.query(PackagesIssue.issue_status).group_by( +- PackagesIssue.issue_status).all() +- return jsonify(ResponseCode.response_json( +- ResponseCode.SUCCESS, [issue_query[0] for issue_query in issues_query])) +- except (SQLAlchemyError, DisconnectionError) as error: +- current_app.logger.error(error) +- return jsonify(ResponseCode.response_json( +- ResponseCode.PARAM_ERROR)) +- +- def get(self): +- """ +- Description: Get all issues info or one specific issue +- Args: +- Returns: +- [ +- "issue_status", +- "issue_status" +- ] +- Raises: +- DisconnectionError: Unable to connect to database exception +- AttributeError: Object does not have this property +- TypeError: Exception of type +- Error: Abnormal error +- """ +- return self._get_issue_status() +- +- +-class IssueCatch(Resource): +- """ +- description: Catch issue content +- Restful API: put +- ChangeLog: +- """ +- +- def post(self): +- """ +- Searching issue content +- Args: +- Returns: +- for examples: +- [ +- { +- "issue_id": "", +- "issue_url": "", +- "issue_title": "", +- "issue_content": "", +- "issue_status": "", +- "issue_type": "" +- }, +- ] +- Raises: +- DisconnectionError: Unable to connect to database exception +- AttributeError: Object does not have this property +- TypeError: Exception of type +- Error: Abnormal error +- """ +- data = json.loads(request.get_data()) +- if not isinstance(data, dict): +- return jsonify( +- ResponseCode.response_json(ResponseCode.PARAM_ERROR)) +- pkg_name = data["repository"]["path"] +- try: +- _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH) +- pool_workers = _readconfig.get_config('LIFECYCLE', 'pool_workers') +- _warehouse = _readconfig.get_config('LIFECYCLE', 'warehouse') +- if _warehouse is None: +- _warehouse = 'src-openeuler' +- if not isinstance(pool_workers, int): +- pool_workers = 10 +- pool = ThreadPoolExecutor(max_workers=pool_workers) +- with DBHelper(db_name="lifecycle") as database: +- for table_name in filter(lambda x: x not in ['packages_issue', 'packages_maintainer'], +- database.engine.table_names()): +- cls_model = Packages.package_meta(table_name) +- for package_item in database.session.query(cls_model).filter( +- cls_model.name == pkg_name).all(): +- gitee_issue = gitee( +- package_item, _warehouse, package_item.name, table_name) +- pool.submit(gitee_issue.issue_hooks, data) +- pool.shutdown() +- return jsonify(ResponseCode.response_json(ResponseCode.SUCCESS)) +- except SQLAlchemyError as error_msg: +- current_app.logger.error(error_msg) +- +- +-class UpdatePackages(Resource): +- """ +- description:Life cycle update information of a single package +- Restful API: post +- ChangeLog: +- """ +- +- def _get_all_yaml_name(self, filepath): +- """ +- List of all yaml file names in the folder +- +- Args: +- filepath: file path +- +- Returns: +- yaml_file_list：List of all yaml file names in the folder +- +- Attributes： +- Error：Error +- NotADirectoryError：Invalid directory name +- FileNotFoundError：File not found error +- +- """ +- try: +- yaml_file_list = os.listdir(filepath) +- return yaml_file_list +- except (Error, NotADirectoryError, FileNotFoundError) as error: +- current_app.logger.error(error) +- return None +- +- def _get_yaml_content(self, yaml_file, filepath): +- """ +- Read the content of the yaml file +- +- Args: +- yaml_file: yaml file +- filepath: file path +- +- Returns: +- Return a dictionary containing name, maintainer and maintainlevel +- """ +- yaml_data_dict = dict() +- if not yaml_file.endswith(".yaml"): +- return None +- pkg_name = yaml_file.rsplit('.yaml')[0] +- single_yaml_path = os.path.join(filepath, yaml_file) +- with open(single_yaml_path, 'r', encoding='utf-8') as file_context: +- yaml_flie_data = yaml.load( +- file_context.read(), Loader=yaml.FullLoader) +- if yaml_flie_data is None or not isinstance(yaml_flie_data, dict): +- return None +- maintainer = yaml_flie_data.get("maintainer") +- maintainlevel = yaml_flie_data.get("maintainlevel") +- yaml_data_dict['name'] = pkg_name +- if maintainer: +- yaml_data_dict['maintainer'] = maintainer +- if maintainlevel: +- yaml_data_dict['maintainlevel'] = maintainlevel +- return yaml_data_dict +- +- def _read_yaml_file(self, filepath): +- """ +- Read the yaml file and combine the data of the nested dictionary of the list +- +- Args: +- filepath: file path +- +- Returns: +- yaml.YAMLError：yaml file error +- SQLAlchemyError：SQLAlchemy Error +- DisconnectionError：Connect to database error +- Error：Error +- """ +- yaml_file_list = self._get_all_yaml_name(filepath) +- if not yaml_file_list: +- return None +- try: +- yaml_data_list = list() +- _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH) +- pool_workers = _readconfig.get_config('LIFECYCLE', 'pool_workers') +- if not isinstance(pool_workers, int): +- pool_workers = 10 +- with ThreadPoolExecutor(max_workers=pool_workers) as pool: +- for yaml_file in yaml_file_list: +- pool_result = pool.submit( +- self._get_yaml_content, yaml_file, filepath) +- yaml_data_dict = pool_result.result() +- yaml_data_list.append(yaml_data_dict) +- return yaml_data_list +- except (yaml.YAMLError, SQLAlchemyError, DisconnectionError, Error) as error: +- current_app.logger.error(error) +- return None +- +- def _verification_yaml_data_list(self, yaml_data_list): +- """ +- Verify the data obtained in the yaml file +- +- Args: +- yaml_data_list: yaml data list +- +- Returns: +- yaml_data_list: After verification yaml data list +- +- Attributes: +- ValidationError: Validation error +- +- """ +- try: +- DataFormatVerfi(many=True).load(yaml_data_list) +- return yaml_data_list +- except ValidationError as error: +- current_app.logger.error(error.messages) +- return None +- +- def _save_in_database(self, yaml_data_list): +- """ +- Save the data to the database +- +- Args: +- tbname: Table Name +- name_separate_list: Split name list +- _update_pack_data: Split new list of combined data +- +- Returns: +- SUCCESS or UPDATA_DATA_FAILED +- +- Attributes +- DisconnectionError: Connect to database error +- SQLAlchemyError: SQLAlchemy Error +- Error: Error +- +- """ +- try: +- with DBHelper(db_name="lifecycle") as database_name: +- if 'packages_maintainer' not in database_name.engine.table_names(): +- return jsonify(ResponseCode.response_json( +- ResponseCode.TABLE_NAME_NOT_EXIST)) +- database_name.session.begin(subtransactions=True) +- for yaml_data in yaml_data_list: +- name = yaml_data.get("name") +- maintainer = yaml_data.get("maintainer") +- maintainlevel = yaml_data.get("maintainlevel") +- packages_maintainer_obj = database_name.session.query( +- PackagesMaintainer).filter_by(name=name).first() +- if packages_maintainer_obj: +- if maintainer: +- packages_maintainer_obj.maintainer = maintainer +- if maintainlevel: +- packages_maintainer_obj.maintainlevel = maintainlevel +- else: +- database_name.add(PackagesMaintainer( +- name=name, maintainer=maintainer, maintainlevel=maintainlevel +- )) +- database_name.session.commit() +- return jsonify(ResponseCode.response_json( +- ResponseCode.SUCCESS)) +- except (DisconnectionError, SQLAlchemyError, Error, AttributeError) as error: +- current_app.logger.error(error) +- return jsonify(ResponseCode.response_json( +- ResponseCode.UPDATA_DATA_FAILED)) +- +- def _overall_process( +- self, +- filepath): +- """ +- Call each method to complete the entire function +- +- Args: +- filepath: file path +- tbname: table name +- +- Returns: +- SUCCESS or UPDATA_DATA_FAILED +- +- Attributes +- DisconnectionError: Connect to database error +- SQLAlchemyError: SQLAlchemy Error +- Error: Error +- """ +- try: +- if filepath is None or not os.path.exists(filepath): +- return jsonify(ResponseCode.response_json( +- ResponseCode.SPECIFIED_FILE_NOT_EXIST)) +- yaml_file_list = self._get_all_yaml_name(filepath) +- if not yaml_file_list: +- return jsonify(ResponseCode.response_json( +- ResponseCode.EMPTY_FOLDER)) +- yaml_data_list_result = self._read_yaml_file(filepath) +- yaml_data_list = self._verification_yaml_data_list( +- yaml_data_list_result) +- if yaml_data_list is None: +- return jsonify(ResponseCode.response_json( +- ResponseCode.YAML_FILE_ERROR)) +- result = self._save_in_database( +- yaml_data_list) +- return result +- except (DisconnectionError, SQLAlchemyError, Error) as error: +- current_app.logger.error(error) +- return jsonify(ResponseCode.response_json( +- ResponseCode.UPDATA_DATA_FAILED)) +- +- def _update_single_package_info( +- self, srcname, maintainer, maintainlevel): +- """ +- Update the maintainer field and maintainlevel +- field of a single package +- +- Args: +- srcname: The name of the source package +- maintainer: Package maintainer +- maintainlevel: Package maintenance level +- +- Returns: +- success or failed +- +- Attributes +- SQLAlchemyError: sqlalchemy error +- DisconnectionError: Cannot connect to database error +- Error: Error +- """ +- if not srcname: +- return jsonify( +- ResponseCode.response_json(ResponseCode.PACK_NAME_NOT_FOUND) +- ) +- if not maintainer and not maintainlevel: +- return jsonify( +- ResponseCode.response_json(ResponseCode.PARAM_ERROR) +- ) +- try: +- with DBHelper(db_name='lifecycle') as database_name: +- if 'packages_maintainer' not in database_name.engine.table_names(): +- return jsonify(ResponseCode.response_json( +- ResponseCode.TABLE_NAME_NOT_EXIST)) +- update_obj = database_name.session.query( +- PackagesMaintainer).filter_by(name=srcname).first() +- if update_obj: +- if maintainer: +- update_obj.maintainer = maintainer +- if maintainlevel: +- update_obj.maintainlevel = maintainlevel +- else: +- database_name.add(PackagesMaintainer( +- name=srcname, maintainer=maintainer, maintainlevel=maintainlevel +- )) +- database_name.session.commit() +- return jsonify( +- ResponseCode.response_json( +- ResponseCode.SUCCESS)) +- except (SQLAlchemyError, DisconnectionError, Error) as sql_error: +- current_app.logger.error(sql_error) +- database_name.session.rollback() +- return jsonify(ResponseCode.response_json( +- ResponseCode.UPDATA_DATA_FAILED +- )) +- +- def put(self): +- """ +- Life cycle update information of a single package or +- All packages +- +- Returns: +- for example:: +- { +- "code": "", +- "data": "", +- "msg": "" +- } +- """ +- schema = UpdatePackagesSchema() +- data = request.get_json() +- if schema.validate(data): +- return jsonify( +- ResponseCode.response_json(ResponseCode.PARAM_ERROR) +- ) +- srcname = data.get('pkg_name', None) +- maintainer = data.get('maintainer', None) +- maintainlevel = data.get('maintainlevel', None) +- batch = data.get('batch') +- filepath = data.get('filepath', None) +- +- if batch: +- result = self._overall_process(filepath) +- else: +- result = self._update_single_package_info( +- srcname, maintainer, maintainlevel) +- return result ++#!/usr/bin/python3 ++""" ++Life cycle related api interface ++""" ++import io ++import json ++import math ++import os ++from concurrent.futures import ThreadPoolExecutor ++ ++import pandas as pd ++import yaml ++ ++from flask import request ++from flask import jsonify, make_response ++from flask import current_app ++from flask_restful import Resource ++from marshmallow import ValidationError ++ ++from sqlalchemy.exc import DisconnectionError, SQLAlchemyError ++ ++from packageship import system_config ++from packageship.libs.configutils.readconfig import ReadConfig ++from packageship.libs.exception import Error ++from packageship.application.apps.package.function.constants import ResponseCode ++from packageship.libs.dbutils.sqlalchemy_helper import DBHelper ++from packageship.application.models.package import PackagesIssue ++from packageship.application.models.package import Packages ++from packageship.application.models.package import PackagesMaintainer ++from packageship.libs.log import Log ++from .serialize import IssueDownloadSchema, PackagesDownloadSchema, IssuePageSchema, IssueSchema ++from ..package.serialize import DataFormatVerfi, UpdatePackagesSchema ++from .function.gitee import Gitee as gitee ++ ++LOGGER = Log(__name__) ++ ++ ++# pylint: disable = no-self-use ++ ++class DownloadFile(Resource): ++ """ ++ Download the content of the issue or the excel file of the package content ++ """ ++ ++ def _download_excel(self, file_type, table_name=None): ++ """ ++ Download excel file ++ """ ++ file_name = 'packages.xlsx' ++ if file_type == 'packages': ++ download_content = self.__get_packages_content(table_name) ++ else: ++ file_name = 'issues.xlsx' ++ download_content = self.__get_issues_content() ++ if download_content is None: ++ return jsonify( ++ ResponseCode.response_json( ++ ResponseCode.SERVICE_ERROR)) ++ pd_dataframe = self.__to_dataframe(download_content) ++ ++ _response = self.__bytes_save(pd_dataframe) ++ return self.__set_response_header(_response, file_name) ++ ++ def __bytes_save(self, data_frame): ++ """ ++ Save the file content in the form of a binary file stream ++ """ ++ try: ++ bytes_io = io.BytesIO() ++ writer = pd.ExcelWriter( # pylint: disable=abstract-class-instantiated ++ bytes_io, engine='xlsxwriter') ++ data_frame.to_excel(writer, sheet_name='Summary', index=False) ++ writer.save() ++ writer.close() ++ bytes_io.seek(0) ++ _response = make_response(bytes_io.getvalue()) ++ bytes_io.close() ++ return _response ++ except (IOError, Error) as io_error: ++ current_app.logger.error(io_error) ++ return make_response() ++ ++ def __set_response_header(self, response, file_name): ++ """ ++ Set http response header information ++ """ ++ response.headers['Content-Type'] = \ ++ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" ++ response.headers["Cache-Control"] = "no-cache" ++ response.headers['Content-Disposition'] = 'attachment; filename={file_name}'.format( ++ file_name=file_name) ++ return response ++ ++ def __get_packages_content(self, table_name): ++ """ ++ Get package list information ++ """ ++ try: ++ with DBHelper(db_name='lifecycle') as database: ++ # Query all package data in the specified table ++ _model = Packages.package_meta(table_name) ++ _packageinfos = database.session.query(_model).all() ++ packages_dicts = PackagesDownloadSchema( ++ many=True).dump(_packageinfos) ++ return packages_dicts ++ ++ except (SQLAlchemyError, DisconnectionError) as error: ++ current_app.logger.error(error) ++ return None ++ ++ def __get_issues_content(self): ++ """ ++ Get the list of issues ++ """ ++ try: ++ with DBHelper(db_name='lifecycle') as database: ++ _issues = database.session.query(PackagesIssue).all() ++ issues_dicts = IssueDownloadSchema(many=True).dump(_issues) ++ return issues_dicts ++ except (SQLAlchemyError, DisconnectionError) as error: ++ current_app.logger.error(error) ++ return None ++ ++ def __to_dataframe(self, datas): ++ """ ++ Convert the obtained information into pandas content format ++ """ ++ ++ data_frame = pd.DataFrame(datas) ++ return data_frame ++ ++ def get(self, file_type): ++ """ ++ Download package collection information and isse list information ++ ++ """ ++ if file_type not in ['packages', 'issues']: ++ return jsonify( ++ ResponseCode.response_json( ++ ResponseCode.PARAM_ERROR)) ++ ++ table_name = request.args.get('table_name', None) ++ response = self._download_excel(file_type, table_name) ++ return response ++ ++ ++class MaintainerView(Resource): ++ """ ++ Maintainer name collection ++ """ ++ ++ def __query_maintainers(self): ++ """ ++ Query the names of all maintainers in the specified table ++ """ ++ try: ++ with DBHelper(db_name='lifecycle') as database: ++ maintainers = database.session.query( ++ PackagesMaintainer.maintainer).group_by(PackagesMaintainer.maintainer).all() ++ return [maintainer_item[0] for maintainer_item in maintainers ++ if maintainer_item[0]] ++ except (SQLAlchemyError, DisconnectionError) as error: ++ current_app.logger.error(error) ++ return [] ++ ++ def get(self): ++ """ ++ Get the list of maintainers ++ """ ++ # Group query of the names of all maintainers in the current table ++ maintainers = self.__query_maintainers() ++ return jsonify(ResponseCode.response_json( ++ ResponseCode.SUCCESS, ++ maintainers)) ++ ++ ++class TableColView(Resource): ++ """ ++ The default column of the package shows the interface ++ """ ++ ++ def __columns_names(self): ++ """ ++ Mapping of column name and title ++ """ ++ columns = [ ++ ('name', 'Name', True), ++ ('version', 'Version', True), ++ ('release', 'Release', True), ++ ('url', 'Url', True), ++ ('rpm_license', 'License', False), ++ ('feature', 'Feature', False), ++ ('maintainer', 'Maintainer', True), ++ ('maintainlevel', 'Maintenance Level', True), ++ ('release_time', 'Release Time', False), ++ ('used_time', 'Used Time', True), ++ ('maintainer_status', 'Maintain Status', True), ++ ('latest_version', 'Latest Version', False), ++ ('latest_version_time', 'Latest Version Release Time', False), ++ ('issue', 'Issue', True)] ++ return columns ++ ++ def __columns_mapping(self): ++ """ ++ ++ """ ++ columns = list() ++ for column in self.__columns_names(): ++ columns.append({ ++ 'column_name': column[0], ++ 'label': column[1], ++ 'default_selected': column[2] ++ }) ++ return columns ++ ++ def get(self): ++ """ ++ Get the default display column of the package ++ ++ """ ++ table_mapping_columns = self.__columns_mapping() ++ return jsonify( ++ ResponseCode.response_json( ++ ResponseCode.SUCCESS, ++ table_mapping_columns)) ++ ++ ++class LifeTables(Resource): ++ """ ++ description: LifeTables ++ Restful API: get ++ ChangeLog: ++ """ ++ ++ def get(self): ++ """ ++ return all table names in the database ++ ++ Returns: ++ Return the table names in the database as a list ++ """ ++ try: ++ with DBHelper(db_name="lifecycle") as database_name: ++ # View all table names in the package-info database ++ all_table_names = database_name.engine.table_names() ++ all_table_names.remove("packages_issue") ++ all_table_names.remove("packages_maintainer") ++ return jsonify( ++ ResponseCode.response_json( ++ ResponseCode.SUCCESS, data=all_table_names) ++ ) ++ except (SQLAlchemyError, DisconnectionError, Error, ValueError) as sql_error: ++ LOGGER.logger.error(sql_error) ++ return jsonify( ++ ResponseCode.response_json(ResponseCode.DATABASE_NOT_FOUND) ++ ) ++ ++ ++class IssueView(Resource): ++ """ ++ Issue content collection ++ """ ++ ++ def _query_issues(self, request_data): ++ """ ++ Args: ++ request_data: ++ Returns: ++ """ ++ try: ++ with DBHelper(db_name='lifecycle') as database: ++ issues_query = database.session.query(PackagesIssue.issue_id, ++ PackagesIssue.issue_url, ++ PackagesIssue.issue_title, ++ PackagesIssue.issue_status, ++ PackagesIssue.pkg_name, ++ PackagesIssue.issue_type, ++ PackagesMaintainer.maintainer). \ ++ outerjoin(PackagesMaintainer, ++ PackagesMaintainer.name == PackagesIssue.pkg_name) ++ if request_data.get("pkg_name"): ++ issues_query = issues_query.filter( ++ PackagesIssue.pkg_name == request_data.get("pkg_name")) ++ if request_data.get("issue_type"): ++ issues_query = issues_query.filter( ++ PackagesIssue.issue_type == request_data.get("issue_type")) ++ if request_data.get("issue_status"): ++ issues_query = issues_query.filter( ++ PackagesIssue.issue_status == request_data.get("issue_status")) ++ if request_data.get("maintainer"): ++ issues_query = issues_query.filter( ++ PackagesMaintainer.maintainer == request_data.get("maintainer")) ++ total_count = issues_query.count() ++ total_page = math.ceil( ++ total_count / int(request_data.get("page_size"))) ++ issues_query = issues_query.limit(request_data.get("page_size")).offset( ++ (int(request_data.get("page_num")) - 1) * int(request_data.get("page_size"))) ++ issue_dicts = IssuePageSchema( ++ many=True).dump(issues_query.all()) ++ issue_data = ResponseCode.response_json( ++ ResponseCode.SUCCESS, issue_dicts) ++ issue_data['total_count'] = total_count ++ issue_data['total_page'] = total_page ++ return issue_data ++ except (SQLAlchemyError, DisconnectionError) as error: ++ current_app.logger.error(error) ++ return ResponseCode.response_json(ResponseCode.DATABASE_NOT_FOUND) ++ ++ def get(self): ++ """ ++ Description: Get all issues info or one specific issue ++ Args: ++ Returns: ++ [ ++ { ++ "issue_id": "", ++ "issue_url": "", ++ "issue_title": "", ++ "issue_content": "", ++ "issue_status": "", ++ "issue_type": "" ++ }, ++ ] ++ Raises: ++ DisconnectionError: Unable to connect to database exception ++ AttributeError: Object does not have this property ++ TypeError: Exception of type ++ Error: Abnormal error ++ """ ++ schema = IssueSchema() ++ if schema.validate(request.args): ++ return jsonify( ++ ResponseCode.response_json(ResponseCode.PARAM_ERROR) ++ ) ++ issue_dict = self._query_issues(request.args) ++ return issue_dict ++ ++ ++class IssueType(Resource): ++ """ ++ Issue type collection ++ """ ++ ++ def _get_issue_type(self): ++ """ ++ Description: Query issue type ++ Returns: ++ """ ++ try: ++ with DBHelper(db_name='lifecycle') as database: ++ issues_query = database.session.query(PackagesIssue.issue_type).group_by( ++ PackagesIssue.issue_type).all() ++ return jsonify(ResponseCode.response_json( ++ ResponseCode.SUCCESS, [issue_query[0] for issue_query in issues_query])) ++ except (SQLAlchemyError, DisconnectionError) as error: ++ current_app.logger.error(error) ++ return jsonify(ResponseCode.response_json( ++ ResponseCode.PARAM_ERROR)) ++ ++ def get(self): ++ """ ++ Description: Get all issues info or one specific issue ++ Args: ++ Returns: ++ [ ++ "issue_type", ++ "issue_type" ++ ] ++ Raises: ++ DisconnectionError: Unable to connect to database exception ++ AttributeError: Object does not have this property ++ TypeError: Exception of type ++ Error: Abnormal error ++ """ ++ return self._get_issue_type() ++ ++ ++class IssueStatus(Resource): ++ """ ++ Issue status collection ++ """ ++ ++ def _get_issue_status(self): ++ """ ++ Description: Query issue status ++ Returns: ++ """ ++ try: ++ with DBHelper(db_name='lifecycle') as database: ++ issues_query = database.session.query(PackagesIssue.issue_status).group_by( ++ PackagesIssue.issue_status).all() ++ return jsonify(ResponseCode.response_json( ++ ResponseCode.SUCCESS, [issue_query[0] for issue_query in issues_query])) ++ except (SQLAlchemyError, DisconnectionError) as error: ++ current_app.logger.error(error) ++ return jsonify(ResponseCode.response_json( ++ ResponseCode.PARAM_ERROR)) ++ ++ def get(self): ++ """ ++ Description: Get all issues info or one specific issue ++ Args: ++ Returns: ++ [ ++ "issue_status", ++ "issue_status" ++ ] ++ Raises: ++ DisconnectionError: Unable to connect to database exception ++ AttributeError: Object does not have this property ++ TypeError: Exception of type ++ Error: Abnormal error ++ """ ++ return self._get_issue_status() ++ ++ ++class IssueCatch(Resource): ++ """ ++ description: Catch issue content ++ Restful API: put ++ ChangeLog: ++ """ ++ ++ def post(self): ++ """ ++ Searching issue content ++ Args: ++ Returns: ++ for examples: ++ [ ++ { ++ "issue_id": "", ++ "issue_url": "", ++ "issue_title": "", ++ "issue_content": "", ++ "issue_status": "", ++ "issue_type": "" ++ }, ++ ] ++ Raises: ++ DisconnectionError: Unable to connect to database exception ++ AttributeError: Object does not have this property ++ TypeError: Exception of type ++ Error: Abnormal error ++ """ ++ data = json.loads(request.get_data()) ++ if not isinstance(data, dict): ++ return jsonify( ++ ResponseCode.response_json(ResponseCode.PARAM_ERROR)) ++ pkg_name = data["repository"]["path"] ++ try: ++ _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH) ++ pool_workers = _readconfig.get_config('LIFECYCLE', 'pool_workers') ++ _warehouse = _readconfig.get_config('LIFECYCLE', 'warehouse') ++ if _warehouse is None: ++ _warehouse = 'src-openeuler' ++ if not isinstance(pool_workers, int): ++ pool_workers = 10 ++ pool = ThreadPoolExecutor(max_workers=pool_workers) ++ with DBHelper(db_name="lifecycle") as database: ++ for table_name in filter(lambda x: x not in ['packages_issue', 'packages_maintainer', 'database_info'], ++ database.engine.table_names()): ++ cls_model = Packages.package_meta(table_name) ++ for package_item in database.session.query(cls_model).filter( ++ cls_model.name == pkg_name).all(): ++ gitee_issue = gitee( ++ package_item, _warehouse, package_item.name, table_name) ++ pool.submit(gitee_issue.issue_hooks, data) ++ pool.shutdown() ++ return jsonify(ResponseCode.response_json(ResponseCode.SUCCESS)) ++ except SQLAlchemyError as error_msg: ++ current_app.logger.error(error_msg) ++ ++ ++class UpdatePackages(Resource): ++ """ ++ description:Life cycle update information of a single package ++ Restful API: post ++ ChangeLog: ++ """ ++ ++ def _get_all_yaml_name(self, filepath): ++ """ ++ List of all yaml file names in the folder ++ ++ Args: ++ filepath: file path ++ ++ Returns: ++ yaml_file_list：List of all yaml file names in the folder ++ ++ Attributes： ++ Error：Error ++ NotADirectoryError：Invalid directory name ++ FileNotFoundError：File not found error ++ ++ """ ++ try: ++ yaml_file_list = os.listdir(filepath) ++ return yaml_file_list ++ except (Error, NotADirectoryError, FileNotFoundError) as error: ++ current_app.logger.error(error) ++ return None ++ ++ def _get_yaml_content(self, yaml_file, filepath): ++ """ ++ Read the content of the yaml file ++ ++ Args: ++ yaml_file: yaml file ++ filepath: file path ++ ++ Returns: ++ Return a dictionary containing name, maintainer and maintainlevel ++ """ ++ yaml_data_dict = dict() ++ if not yaml_file.endswith(".yaml"): ++ return None ++ pkg_name = yaml_file.rsplit('.yaml')[0] ++ single_yaml_path = os.path.join(filepath, yaml_file) ++ with open(single_yaml_path, 'r', encoding='utf-8') as file_context: ++ yaml_flie_data = yaml.load( ++ file_context.read(), Loader=yaml.FullLoader) ++ if yaml_flie_data is None or not isinstance(yaml_flie_data, dict): ++ return None ++ maintainer = yaml_flie_data.get("maintainer") ++ maintainlevel = yaml_flie_data.get("maintainlevel") ++ yaml_data_dict['name'] = pkg_name ++ if maintainer: ++ yaml_data_dict['maintainer'] = maintainer ++ if maintainlevel: ++ yaml_data_dict['maintainlevel'] = maintainlevel ++ return yaml_data_dict ++ ++ def _read_yaml_file(self, filepath): ++ """ ++ Read the yaml file and combine the data of the nested dictionary of the list ++ ++ Args: ++ filepath: file path ++ ++ Returns: ++ yaml.YAMLError：yaml file error ++ SQLAlchemyError：SQLAlchemy Error ++ DisconnectionError：Connect to database error ++ Error：Error ++ """ ++ yaml_file_list = self._get_all_yaml_name(filepath) ++ if not yaml_file_list: ++ return None ++ try: ++ yaml_data_list = list() ++ _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH) ++ pool_workers = _readconfig.get_config('LIFECYCLE', 'pool_workers') ++ if not isinstance(pool_workers, int): ++ pool_workers = 10 ++ with ThreadPoolExecutor(max_workers=pool_workers) as pool: ++ for yaml_file in yaml_file_list: ++ pool_result = pool.submit( ++ self._get_yaml_content, yaml_file, filepath) ++ yaml_data_dict = pool_result.result() ++ yaml_data_list.append(yaml_data_dict) ++ return yaml_data_list ++ except (yaml.YAMLError, SQLAlchemyError, DisconnectionError, Error) as error: ++ current_app.logger.error(error) ++ return None ++ ++ def _verification_yaml_data_list(self, yaml_data_list): ++ """ ++ Verify the data obtained in the yaml file ++ ++ Args: ++ yaml_data_list: yaml data list ++ ++ Returns: ++ yaml_data_list: After verification yaml data list ++ ++ Attributes: ++ ValidationError: Validation error ++ ++ """ ++ try: ++ DataFormatVerfi(many=True).load(yaml_data_list) ++ return yaml_data_list ++ except ValidationError as error: ++ current_app.logger.error(error.messages) ++ return None ++ ++ def _save_in_database(self, yaml_data_list): ++ """ ++ Save the data to the database ++ ++ Args: ++ tbname: Table Name ++ name_separate_list: Split name list ++ _update_pack_data: Split new list of combined data ++ ++ Returns: ++ SUCCESS or UPDATA_DATA_FAILED ++ ++ Attributes ++ DisconnectionError: Connect to database error ++ SQLAlchemyError: SQLAlchemy Error ++ Error: Error ++ ++ """ ++ try: ++ with DBHelper(db_name="lifecycle") as database_name: ++ if 'packages_maintainer' not in database_name.engine.table_names(): ++ return jsonify(ResponseCode.response_json( ++ ResponseCode.TABLE_NAME_NOT_EXIST)) ++ database_name.session.begin(subtransactions=True) ++ for yaml_data in yaml_data_list: ++ name = yaml_data.get("name") ++ maintainer = yaml_data.get("maintainer") ++ maintainlevel = yaml_data.get("maintainlevel") ++ packages_maintainer_obj = database_name.session.query( ++ PackagesMaintainer).filter_by(name=name).first() ++ if packages_maintainer_obj: ++ if maintainer: ++ packages_maintainer_obj.maintainer = maintainer ++ if maintainlevel: ++ packages_maintainer_obj.maintainlevel = maintainlevel ++ else: ++ database_name.add(PackagesMaintainer( ++ name=name, maintainer=maintainer, maintainlevel=maintainlevel ++ )) ++ database_name.session.commit() ++ return jsonify(ResponseCode.response_json( ++ ResponseCode.SUCCESS)) ++ except (DisconnectionError, SQLAlchemyError, Error, AttributeError) as error: ++ current_app.logger.error(error) ++ return jsonify(ResponseCode.response_json( ++ ResponseCode.UPDATA_DATA_FAILED)) ++ ++ def _overall_process( ++ self, ++ filepath): ++ """ ++ Call each method to complete the entire function ++ ++ Args: ++ filepath: file path ++ tbname: table name ++ ++ Returns: ++ SUCCESS or UPDATA_DATA_FAILED ++ ++ Attributes ++ DisconnectionError: Connect to database error ++ SQLAlchemyError: SQLAlchemy Error ++ Error: Error ++ """ ++ try: ++ if filepath is None or not os.path.exists(filepath): ++ return jsonify(ResponseCode.response_json( ++ ResponseCode.SPECIFIED_FILE_NOT_EXIST)) ++ yaml_file_list = self._get_all_yaml_name(filepath) ++ if not yaml_file_list: ++ return jsonify(ResponseCode.response_json( ++ ResponseCode.EMPTY_FOLDER)) ++ yaml_data_list_result = self._read_yaml_file(filepath) ++ yaml_data_list = self._verification_yaml_data_list( ++ yaml_data_list_result) ++ if yaml_data_list is None: ++ return jsonify(ResponseCode.response_json( ++ ResponseCode.YAML_FILE_ERROR)) ++ result = self._save_in_database( ++ yaml_data_list) ++ return result ++ except (DisconnectionError, SQLAlchemyError, Error) as error: ++ current_app.logger.error(error) ++ return jsonify(ResponseCode.response_json( ++ ResponseCode.UPDATA_DATA_FAILED)) ++ ++ def _update_single_package_info( ++ self, srcname, maintainer, maintainlevel): ++ """ ++ Update the maintainer field and maintainlevel ++ field of a single package ++ ++ Args: ++ srcname: The name of the source package ++ maintainer: Package maintainer ++ maintainlevel: Package maintenance level ++ ++ Returns: ++ success or failed ++ ++ Attributes ++ SQLAlchemyError: sqlalchemy error ++ DisconnectionError: Cannot connect to database error ++ Error: Error ++ """ ++ if not srcname: ++ return jsonify( ++ ResponseCode.response_json(ResponseCode.PACK_NAME_NOT_FOUND) ++ ) ++ if not maintainer and not maintainlevel: ++ return jsonify( ++ ResponseCode.response_json(ResponseCode.PARAM_ERROR) ++ ) ++ try: ++ with DBHelper(db_name='lifecycle') as database_name: ++ if 'packages_maintainer' not in database_name.engine.table_names(): ++ return jsonify(ResponseCode.response_json( ++ ResponseCode.TABLE_NAME_NOT_EXIST)) ++ update_obj = database_name.session.query( ++ PackagesMaintainer).filter_by(name=srcname).first() ++ if update_obj: ++ if maintainer: ++ update_obj.maintainer = maintainer ++ if maintainlevel: ++ update_obj.maintainlevel = maintainlevel ++ else: ++ database_name.add(PackagesMaintainer( ++ name=srcname, maintainer=maintainer, maintainlevel=maintainlevel ++ )) ++ database_name.session.commit() ++ return jsonify( ++ ResponseCode.response_json( ++ ResponseCode.SUCCESS)) ++ except (SQLAlchemyError, DisconnectionError, Error) as sql_error: ++ current_app.logger.error(sql_error) ++ database_name.session.rollback() ++ return jsonify(ResponseCode.response_json( ++ ResponseCode.UPDATA_DATA_FAILED ++ )) ++ ++ def put(self): ++ """ ++ Life cycle update information of a single package or ++ All packages ++ ++ Returns: ++ for example:: ++ { ++ "code": "", ++ "data": "", ++ "msg": "" ++ } ++ """ ++ schema = UpdatePackagesSchema() ++ data = request.get_json() ++ if schema.validate(data): ++ return jsonify( ++ ResponseCode.response_json(ResponseCode.PARAM_ERROR) ++ ) ++ srcname = data.get('pkg_name', None) ++ maintainer = data.get('maintainer', None) ++ maintainlevel = data.get('maintainlevel', None) ++ batch = data.get('batch') ++ filepath = data.get('filepath', None) ++ ++ if batch: ++ result = self._overall_process(filepath) ++ else: ++ result = self._update_single_package_info( ++ srcname, maintainer, maintainlevel) ++ return result +diff -Naru a/packageship/application/apps/package/function/be_depend.py b/packageship/application/apps/package/function/be_depend.py +--- a/packageship/application/apps/package/function/be_depend.py 2020-09-22 23:34:04.037937224 +0800 ++++ b/packageship/application/apps/package/function/be_depend.py 2020-09-22 23:48:32.402476132 +0800 +@@ -5,11 +5,12 @@ + This includes both install and build dependencies + Class: BeDepend + """ ++import copy ++from collections import namedtuple, defaultdict + from flask import current_app + from sqlalchemy import text + from sqlalchemy.exc import SQLAlchemyError + from sqlalchemy.sql import literal_column +-from packageship.application.apps.package.function.constants import ResponseCode + from packageship.application.models.package import SrcPack + from packageship.libs.dbutils import DBHelper + +@@ -36,6 +37,8 @@ + self.source_name_set = set() + self.bin_name_set = set() + self.result_dict = dict() ++ self.comm_install_builds = defaultdict(set) ++ self.provides_name = set() + + def main(self): + """ +@@ -69,14 +72,16 @@ + [["root", None]] + ] + self.source_name_set.add(self.source_name) +- self.package_bedepend( ++ self._provides_bedepend( + [self.source_name], data_base, package_type='src') + ++ for _, value in self.result_dict.items(): ++ value[-1] = list(value[-1]) + return self.result_dict + +- def package_bedepend(self, pkg_name_list, data_base, package_type): ++ def _get_provides(self, pkg_name_list, data_base, package_type): + """ +- Description: Query the dependent function ++ Description: Query the components provided by the required package + Args: + pkg_name_list:source or binary packages name + data_base: database +@@ -84,35 +89,31 @@ + Returns: + Raises: + SQLAlchemyError: Database connection exception +- """ +- ++ """ ++ res = namedtuple( ++ 'restuple', [ ++ 'search_bin_name', 'search_bin_version', 'source_name']) + sql_com = """ +- SELECT DISTINCT b1.name AS search_bin_name, ++ SELECT DISTINCT b1.name AS search_bin_name, + b1.version AS search_bin_version, + b1.src_name AS source_name, +- b2.name AS bin_name, +- s1.name AS bebuild_src_name, +- b2.src_name AS install_depend_src_name ++ bin_provides.name As pro_name + FROM ( SELECT pkgKey,src_name,name,version FROM bin_pack WHERE {} ) b1 +- LEFT JOIN bin_provides ON bin_provides.pkgKey = b1.pkgKey +- LEFT JOIN bin_requires br ON br.name = bin_provides.name +- LEFT JOIN src_requires sr ON sr.name = bin_provides.name +- LEFT JOIN src_pack s1 ON s1.pkgKey = sr.pkgKey +- LEFT JOIN bin_pack b2 ON b2.pkgKey = br.pkgKey +- """ ++ LEFT JOIN bin_provides ON bin_provides.pkgKey = b1.pkgKey;""" + ++ # package_type + if package_type == 'src': + literal_name = 'src_name' +- + elif package_type == 'bin': + literal_name = 'name' + +- else: +- return +- ++ # Query database ++ # The lower version of SQLite can look up up to 999 parameters ++ # simultaneously, so use 900 sharding queries + try: + result = [] +- for input_name in (pkg_name_list[i:i+900] for i in range(0, len(pkg_name_list), 900)): ++ for input_name in (pkg_name_list[i:i + 900] ++ for i in range(0, len(pkg_name_list), 900)): + name_in = literal_column(literal_name).in_(input_name) + sql_str = text(sql_com.format(name_in)) + result.extend(data_base.session.execute( +@@ -124,74 +125,176 @@ + ).fetchall()) + except SQLAlchemyError as sql_err: + current_app.logger.error(sql_err) +- return ResponseCode.response_json(ResponseCode.CONNECT_DB_ERROR) ++ return + + if not result: + return + +- # Source and binary packages that were found to be dependent +- source_name_list = [] +- bin_name_list = [] ++ # Process the result of the component ++ pro_name_dict = dict() ++ ++ _components = set() + for obj in result: +- if obj.source_name is None: +- source_name = 'NOT FOUND' +- else: +- source_name = obj.source_name +- if obj.bebuild_src_name: +- # Determine if the source package has been checked +- parent_node = obj.bebuild_src_name +- be_type = "build" +- # Call the spell dictionary function +- self.make_dicts( +- obj.search_bin_name, +- source_name, ++ if not obj.pro_name: ++ continue ++ # De-weight components ++ if obj.pro_name not in self.comm_install_builds: ++ pro_name_dict[obj.pro_name] = res( ++ obj.search_bin_name, obj.search_bin_version, obj.source_name) ++ ++ if obj.search_bin_name not in self.result_dict: ++ self.result_dict[obj.search_bin_name] = [ ++ obj.source_name, + obj.search_bin_version, +- parent_node, +- be_type) ++ self.db_name, ++ self.comm_install_builds[obj.pro_name] ++ if self.comm_install_builds[obj.pro_name] else {(None, None)} ++ ] ++ tmp_ = copy.deepcopy(self.comm_install_builds[obj.pro_name]) + +- if obj.bebuild_src_name not in self.source_name_set: +- self.source_name_set.add(obj.bebuild_src_name) +- source_name_list.append(obj.bebuild_src_name) +- +- if obj.bin_name: +- # Determine if the bin package has been checked +- parent_node = obj.bin_name +- be_type = "install" +- # Call the spell dictionary function +- self.make_dicts( +- obj.search_bin_name, +- source_name, +- obj.search_bin_version, +- parent_node, +- be_type) ++ tmp_.discard((obj.search_bin_name, 'install')) ++ tmp_.discard((obj.search_bin_name, 'build')) + +- if obj.bin_name not in self.bin_name_set: +- self.bin_name_set.add(obj.bin_name) +- bin_name_list.append(obj.bin_name) +- +- # With_sub_pack=1 +- if self.with_sub_pack == "1": +- if obj.install_depend_src_name not in self.source_name_set: +- self.source_name_set.add( +- obj.install_depend_src_name) +- source_name_list.append( +- obj.install_depend_src_name) +- +- if obj.bebuild_src_name is None and obj.bin_name is None: +- parent_node = None +- be_type = None +- self.make_dicts( +- obj.search_bin_name, +- source_name, +- obj.search_bin_version, +- parent_node, +- be_type) ++ if (None, None) in self.result_dict[obj.search_bin_name][-1] \ ++ and self.comm_install_builds[obj.pro_name]: ++ self.result_dict[obj.search_bin_name][-1] = tmp_ ++ else: ++ self.result_dict[obj.search_bin_name][-1].update(tmp_) ++ return pro_name_dict ++ ++ def _provides_bedepend(self, pkg_name_list, data_base, package_type): ++ """ ++ Description: Query the dependent function ++ Args: ++ pkg_name_list:source or binary packages name ++ data_base: database ++ package_type: package type ++ Returns: ++ Raises: ++ SQLAlchemyError: Database connection exception ++ """ ++ # Query component ++ pro_names = self._get_provides(pkg_name_list, data_base, package_type) + +- if len(source_name_list) != 0: +- self.package_bedepend( ++ if not pro_names: ++ return ++ ++ sql_2_bin = """ ++ SELECT DISTINCT ++ b2.name AS bin_name, ++ b2.src_name AS install_depend_src_name, ++ br.name AS pro_name ++ FROM ++ ( SELECT name, pkgKey FROM bin_requires WHERE {}) br ++ LEFT JOIN bin_pack b2 ON b2.pkgKey = br.pkgKey; ++ """ ++ ++ sql_2_src = """ ++ SELECT DISTINCT ++ s1.name AS bebuild_src_name, ++ sr.name AS pro_name ++ FROM ++ ( SELECT name, pkgKey FROM src_requires WHERE {} ) sr ++ LEFT JOIN src_pack s1 ON s1.pkgKey = sr.pkgKey; ++ """ ++ ++ provides_name_list = [pro for pro, _ in pro_names.items()] ++ ++ result_2_bin = [] ++ result_2_src = [] ++ # Query database ++ try: ++ for input_name in ( ++ provides_name_list[i:i + 900] for i in range(0, len(provides_name_list), 900)): ++ name_in = literal_column('name').in_(input_name) ++ sql_str_2_bin = text(sql_2_bin.format(name_in)) ++ result_2_bin.extend(data_base.session.execute( ++ sql_str_2_bin, ++ { ++ 'name_{}'.format(i): v ++ for i, v in enumerate(input_name, 1) ++ } ++ ).fetchall()) ++ sql_str_2src = text(sql_2_src.format(name_in)) ++ result_2_src.extend(data_base.session.execute( ++ sql_str_2src, ++ { ++ 'name_{}'.format(i): v ++ for i, v in enumerate(input_name, 1) ++ } ++ ).fetchall()) ++ except SQLAlchemyError as sql_err: ++ current_app.logger.error(sql_err) ++ return ++ ++ source_name_list = [] ++ bin_name_list = [] ++ ++ # Process the data that the installation depends on ++ for bin_info in result_2_bin: ++ temp_bin_pkg = bin_info.bin_name ++ temp_sub_src_pkg = bin_info.install_depend_src_name ++ ++ #withsubpick ==1 ++ if self.with_sub_pack == '1' and temp_sub_src_pkg not in self.source_name_set: ++ self.source_name_set.add(temp_sub_src_pkg) ++ source_name_list.append(temp_sub_src_pkg) ++ ++ if temp_bin_pkg not in self.bin_name_set: ++ self.bin_name_set.add(temp_bin_pkg) ++ bin_name_list.append(temp_bin_pkg) ++ ++ if bin_info.pro_name not in self.comm_install_builds: ++ self.comm_install_builds[bin_info.pro_name] = { ++ (bin_info.bin_name, 'install') ++ } ++ ++ elif (bin_info.bin_name, 'install') not in \ ++ self.comm_install_builds[bin_info.pro_name]: ++ ++ self.comm_install_builds[bin_info.pro_name].add( ++ (bin_info.bin_name, 'install') ++ ) ++ ++ self.make_dicts( ++ pro_names.get(bin_info.pro_name).search_bin_name, ++ pro_names.get(bin_info.pro_name).source_name, ++ pro_names.get(bin_info.pro_name).search_bin_version, ++ bin_info.bin_name, ++ 'install' ++ ) ++ # Process data that is compile-dependent ++ for src_info in result_2_src: ++ if src_info.bebuild_src_name not in self.source_name_set: ++ self.source_name_set.add(src_info.bebuild_src_name) ++ source_name_list.append(src_info.bebuild_src_name) ++ ++ if src_info.pro_name not in self.comm_install_builds: ++ self.comm_install_builds[src_info.pro_name] = { ++ (src_info.bebuild_src_name, 'build') ++ } ++ elif (src_info.bebuild_src_name, 'build') not in \ ++ self.comm_install_builds[src_info.pro_name]: ++ ++ self.comm_install_builds[src_info.pro_name].add( ++ (src_info.bebuild_src_name, 'build') ++ ) ++ ++ self.make_dicts( ++ pro_names.get(src_info.pro_name).search_bin_name, ++ pro_names.get(src_info.pro_name).source_name, ++ pro_names.get(src_info.pro_name).search_bin_version, ++ src_info.bebuild_src_name, ++ 'build' ++ ) ++ # Recursively query all source packages that need to be looked up ++ if source_name_list: ++ self._provides_bedepend( + source_name_list, data_base, package_type="src") +- if len(bin_name_list) != 0: +- self.package_bedepend(bin_name_list, data_base, package_type="bin") ++ # Recursively query all binary packages that need to be looked up ++ if bin_name_list: ++ self._provides_bedepend( ++ bin_name_list, data_base, package_type="bin") + + def make_dicts(self, key, source_name, version, parent_node, be_type): + """ +@@ -210,29 +313,27 @@ + source_name, + version, + self.db_name, +- [ +- [parent_node, ++ { ++ (parent_node, + be_type +- ] +- ] ++ ) ++ } ++ + ] + else: + if be_type and parent_node: +- if [None, None] in self.result_dict[key][-1]: +- self.result_dict.pop(key) +- self.result_dict[key] = [ +- source_name, +- version, +- self.db_name, +- [ +- [parent_node, +- be_type +- ] +- ] +- ] ++ if (None, None) in self.result_dict[key][-1]: ++ self.result_dict[key][-1] = { ++ ( ++ parent_node, ++ be_type ++ ) ++ } + +- elif [parent_node, be_type] not in self.result_dict[key][-1]: +- self.result_dict[key][-1].append([ +- parent_node, +- be_type +- ]) ++ elif (parent_node, be_type) not in self.result_dict[key][-1]: ++ self.result_dict[key][-1].add( ++ ( ++ parent_node, ++ be_type ++ ) ++ ) +diff -Naru a/packageship/libs/dbutils/sqlalchemy_helper.py b/packageship/libs/dbutils/sqlalchemy_helper.py +--- a/packageship/libs/dbutils/sqlalchemy_helper.py 2020-09-22 23:34:04.037937224 +0800 ++++ b/packageship/libs/dbutils/sqlalchemy_helper.py 2020-09-22 23:52:23.031681622 +0800 +@@ -9,6 +9,7 @@ + from sqlalchemy.orm import sessionmaker + from sqlalchemy.exc import SQLAlchemyError + from sqlalchemy.exc import DisconnectionError ++from sqlalchemy.exc import OperationalError + from sqlalchemy.ext.declarative import declarative_base + from sqlalchemy.engine.url import URL + from packageship.libs.exception.ext import Error +@@ -252,6 +253,8 @@ + + except SQLAlchemyError as sql_error: + self.session.rollback() ++ if isinstance(sql_error, OperationalError): ++ raise OperationalError + raise Error(sql_error) + else: + self.session.commit() +diff -Naru a/packageship/package.ini b/packageship/package.ini +--- a/packageship/package.ini 2020-09-22 23:34:04.037937224 +0800 ++++ b/packageship/package.ini 2020-09-22 23:49:12.154683915 +0800 +@@ -93,3 +93,5 @@ + ; When this value is not set, the system will default to src-openeuler + warehouse=src-openeuler + ++; Maximum queue length ++queue_maxsize = 1000 +\ No newline at end of file +diff -Naru a/packageship/pkgshipd b/packageship/pkgshipd +--- a/packageship/pkgshipd 2020-09-22 23:34:04.037937224 +0800 ++++ b/packageship/pkgshipd 2020-09-22 23:51:57.323547247 +0800 +@@ -1,6 +1,18 @@ + #!/bin/bash + SYS_PATH=/etc/pkgship + OUT_PATH=/var/run/pkgship_uwsgi ++ ++MEM_THRESHOLD='700' ++MEM_FREE=`free -m | grep "Mem" | awk '{print $7}'` ++ ++if [ $1 = "start" ] ++then ++ if [ $MEM_FREE -lt $MEM_THRESHOLD ]; then ++ echo "[ERROR] pkgship tool does not support memory less than ${MEM_THRESHOLD} MB." ++ exit 0 ++ fi ++fi ++ + if [ ! -d "$OUT_PATH" ]; then + mkdir $OUT_PATH + fi +diff -Naru a/test/common_files/package.ini b/test/common_files/package.ini +--- a/test/common_files/package.ini 2020-09-22 23:34:04.041937245 +0800 ++++ b/test/common_files/package.ini 2020-09-22 23:50:56.559229634 +0800 +@@ -1,30 +1,31 @@ +-[SYSTEM] +-init_conf_path = C:\Users\TAO\Desktop\pkgship-1.1.0\test\common_files\conf.yaml +-write_port = 8080 +-query_port = 8090 +-write_ip_addr = 127.0.0.1 +-query_ip_addr = 127.0.0.1 +-remote_host = https://api.openeuler.org/pkgmanage +- +-[LOG] +-log_level = INFO +-log_name = log_info.log +-backup_count = 10 +-max_bytes = 314572800 +- +-[UWSGI] +-daemonize = /var/log/uwsgi.log +-buffer-size = 65536 +-http-timeout = 600 +-harakiri = 600 +- +-[TIMEDTASK] +-open = True +-hour = 3 +-minute = 0 +- +-[LIFECYCLE] +-warehouse_remote = https://gitee.com/openeuler/openEuler-Advisor/raw/master/upstream-info/ +-pool_workers = 10 +-warehouse = src-openeuler +- ++[SYSTEM] ++init_conf_path = ++write_port = 8080 ++query_port = 8090 ++write_ip_addr = 127.0.0.1 ++query_ip_addr = 127.0.0.1 ++remote_host = https://api.openeuler.org/pkgmanage ++ ++[LOG] ++log_level = INFO ++log_name = log_info.log ++backup_count = 10 ++max_bytes = 314572800 ++ ++[UWSGI] ++daemonize = /var/log/uwsgi.log ++buffer-size = 65536 ++http-timeout = 600 ++harakiri = 600 ++ ++[TIMEDTASK] ++open = True ++hour = 3 ++minute = 0 ++ ++[LIFECYCLE] ++warehouse_remote = https://gitee.com/openeuler/openEuler-Advisor/raw/master/upstream-info/ ++pool_workers = 10 ++warehouse = src-openeuler ++queue_maxsize = 1000 ++ diff --git a/pkgship.spec b/pkgship.spec index edc14f7..a7077d0 100644 --- a/pkgship.spec +++ b/pkgship.spec @@ -1,6 +1,6 @@ Name: pkgship Version: 1.1.0 -Release: 4 +Release: 9 Summary: Pkgship implements rpm package dependence ,maintainer, patch query and so no. License: Mulan 2.0 URL: https://gitee.com/openeuler/openEuler-Advisor @@ -10,6 +10,23 @@ Source0: https://gitee.com/openeuler/openEuler-Advisor/pkgship-%{version} # extract multiplexing functions, add corresponding docString, and clear pylint Patch0: 0001-solve-installation-dependency-query-error.patch +# Fix the problem of continuous spaces in message information in log records +Patch1: 0002-fix-the-problem-of-continuous-spaces.patch + +# When initializing logging, modify the incoming class object to an instance of the class, +# ensure the execution of internal functions,and read configuration file content +Patch2: 0003-fix-log_level-configuration-item-not-work.patch + +# Fix the error when executing query commands +Patch3: 0004-fix-the-error-when-executing-query-commands.patch + +# Add the judgment of whether the subpack_name attribute exists, fix the code indentation problem, +# and reduce the judgment branch of the old code. +Patch4: 0005-fix-the-error-when-source-package-has-no-sub-packages.patch + +# Solve the problem of data duplication, increase the maximum queue length judgment, +# and avoid occupying too much memory +Patch5: 0006-fix-data-duplication-issue.patch BuildArch: noarch BuildRequires: python3-flask-restful python3-flask python3 python3-pyyaml python3-sqlalchemy @@ -58,8 +75,27 @@ rm -rf $log_path %attr(0755,root,root) %{_bindir}/pkgship %changelog +* Tue Sep 21 2020 Shenmei Tu - 1.0-0-9 +- Solve the problem of data duplication, increase the maximum queue length judgment, +- and avoid occupying too much memory + +* Mon Sep 21 2020 Shenmei Tu - 1.0-0-8 +- Add the judgment of whether the subpack_name attribute exists, fix the code indentation problem, +- and reduce the judgment branch of the old code. + +* Mon Sep 21 2020 Shenmei Tu - 1.0-0-7 +- fix the error when executing query commands + +* Mon Sep 21 2020 Shenmei Tu - 1.0-0-6 +- When initializing logging, modify the incoming class object to an instance of the class, +- ensure the execution of internal functions,and read configuration file content + +* Mon Sep 21 2020 Shenmei Tu - 1.0-0-5 +- Fix the problem of continuous spaces in message information in log records + * Thu Sep 17 2020 Shenmei Tu - 1.0-0-4 -- Modify the query logic of package information, reduce redundant queries and align dnf query results, extract multiplexing functions, add corresponding docString, and clear pylint +- Modify the query logic of package information, reduce redundant queries and align dnf query results, +- extract multiplexing functions, add corresponding docString, and clear pylint * Fri Sep 11 2020 Yiru Wang - 1.1.0-3 - #I1UCM8, #I1UC8G: Modify some config files' permission issue;