fix the problem of continuous spaces, fix log_level configuration item not work, fix the error when executing query commands, fix the error when source package has no sub packages, fix data duplication issue

0002-fix-the-problem-of-continuous-spaces.patch

@@ -0,0 +1,255 @@
+diff --git a/packageship/application/initsystem/data_import.py b/packageship/application/initsystem/data_import.py
+index c2169c1..a5846bd 100644
+--- a/packageship/application/initsystem/data_import.py
++++ b/packageship/application/initsystem/data_import.py
+@@ -84,8 +84,8 @@ class InitDataBase():
+ 
+         if not os.path.exists(self.config_file_path):
+             raise FileNotFoundError(
+-                'system initialization configuration file \
+-                    does not exist: %s' % self.config_file_path)
++                "system initialization configuration file"
++                "does not exist: %s" % self.config_file_path)
+         # load yaml configuration file
+         with open(self.config_file_path, 'r', encoding='utf-8') as file_context:
+             try:
+@@ -93,24 +93,25 @@ class InitDataBase():
+                     file_context.read(), Loader=yaml.FullLoader)
+             except yaml.YAMLError as yaml_error:
+ 
+-                raise ConfigurationException(' '.join("The format of the yaml configuration\
+-                     file is wrong please check and try again:{0}".format(yaml_error).split()))
++                raise ConfigurationException(
++                    "The format of the yaml configuration"
++                    "file is wrong please check and try again:{0}".format(yaml_error))
+ 
+             if init_database_config is None:
+                 raise ConfigurationException(
+                     'The content of the database initialization configuration file cannot be empty')
+             if not isinstance(init_database_config, list):
+                 raise ConfigurationException(
+-                    ' '.join('The format of the initial database configuration file\
+-                        is incorrect.When multiple databases need to be initialized, \
+-                        it needs to be configured in the form of multiple \
+-                        nodes:{}'.format(self.config_file_path).split()))
++                    "The format of the initial database configuration file"
++                    "is incorrect.When multiple databases need to be initialized,"
++                    "it needs to be configured in the form of multiple"
++                    "nodes:{}".format(self.config_file_path))
+             for config_item in init_database_config:
+                 if not isinstance(config_item, dict):
+-                    raise ConfigurationException(' '.join('The format of the initial database\
+-                        configuration file is incorrect, and the value in a single node should\
+-                        be presented in the form of key - val pairs: \
+-                        {}'.format(self.config_file_path).split()))
++                    raise ConfigurationException(
++                        "The format of the initial database"
++                        "configuration file is incorrect, and the value in a single node should"
++                        "be presented in the form of key - val pairs:{}".format(self.config_file_path))
+             return init_database_config
+ 
+     def init_data(self):
+@@ -122,8 +123,8 @@ class InitDataBase():
+         """
+         if getattr(self, 'config_file_datas', None) is None or \
+                 self.config_file_datas is None:
+-            raise ContentNoneException('The content of the database initialization \
+-                configuration file is empty')
++            raise ContentNoneException("The content of the database initialization"
++                                       "configuration file is empty")
+ 
+         if self.__exists_repeat_database():
+             raise DatabaseRepeatException(
+@@ -139,13 +140,13 @@ class InitDataBase():
+                 continue
+             priority = database_config.get('priority')
+             if not isinstance(priority, int) or priority < 0 or priority > 100:
+-                LOGGER.logger.error('The priority value type in the database initialization \
+-                    configuration file is incorrect')
++                LOGGER.logger.error("The priority value type in the database initialization"
++                                    "configuration file is incorrect")
+                 continue
+             lifecycle_status_val = database_config.get('lifecycle')
+             if lifecycle_status_val not in ('enable', 'disable'):
+-                LOGGER.logger.error('The status value of the life cycle in the initialization\
+-                    configuration file can only be enable or disable')
++                LOGGER.logger.error("The value of the life cycle in the initialization"
++                                    "configuration file can only be enable or disable")
+                 continue
+             # Initialization data
+             self._init_data(database_config)
+@@ -163,8 +164,8 @@ class InitDataBase():
+         """
+         _database_engine = self._database_engine.get(self.db_type)
+         if not _database_engine:
+-            raise Error('The database engine is set incorrectly, \
+-            currently only the following engines are supported: %s '
++            raise Error("The database engine is set incorrectly,"
++                        "currently only the following engines are supported: %s "
+                         % '、'.join(self._database_engine.keys()))
+         _create_table_result = _database_engine(
+             db_name=db_name, tables=tables, storage=storage).create_database(self)
+@@ -200,11 +201,12 @@ class InitDataBase():
+ 
+             if src_db_file is None or bin_db_file is None:
+                 raise ContentNoneException(
+-                    'The path to the sqlite file in the database initialization configuration \
+-                    is incorrect ')
++                    "The path to the sqlite file in the database initialization"
++                    "configuration is incorrect ")
+             if not os.path.exists(src_db_file) or not os.path.exists(bin_db_file):
+-                raise FileNotFoundError("sqlite file {src} or {bin} does not exist, please \
+-                    check and try again".format(src=src_db_file, bin=bin_db_file))
++                raise FileNotFoundError(
++                    "sqlite file {src} or {bin} does not exist, please"
++                    "check and try again".format(src=src_db_file, bin=bin_db_file))
+             # 3. Obtain temporary source package files and binary package files
+             if self.__save_data(database_config,
+                                 self.database_name):
+@@ -314,23 +316,20 @@ class InitDataBase():
+ 
+         Args:
+             db_name: Saved database name
+-        Returns:
+-
+-        Raises:
+-
+         """
+         # Query all source packages
+         self.sql = " select * from packages "
+         packages_datas = self.__get_data()
+         if packages_datas is None:
+             raise ContentNoneException(
+-                '{db_name}:There is no relevant data in the source \
+-                    package provided '.format(db_name=db_name))
++                "{db_name}:There is no relevant data in the source "
++                "package provided ".format(db_name=db_name))
+         for index, src_package_item in enumerate(packages_datas):
+             try:
+                 src_package_name = '-'.join([src_package_item.get('name'),
+                                              src_package_item.get('version'),
+-                                             src_package_item.get('release') + '.src.rpm'
++                                             src_package_item.get(
++                                                 'release') + '.src.rpm'
+                                              ])
+             except AttributeError as exception_msg:
+                 src_package_name = None
+@@ -391,8 +390,9 @@ class InitDataBase():
+         self.sql = " select * from requires "
+         requires_datas = self.__get_data()
+         if requires_datas is None:
+-            raise ContentNoneException('{db_name}: The package data that the source package \
+-                depends on is empty'.format(db_name=db_name))
++            raise ContentNoneException(
++                "{db_name}: The package data that the source package "
++                "depends on is empty".format(db_name=db_name))
+         with DBHelper(db_name=db_name) as database:
+             database.batch_add(requires_datas, SrcRequires)
+ 
+@@ -411,8 +411,8 @@ class InitDataBase():
+         bin_packaegs = self.__get_data()
+         if bin_packaegs is None:
+             raise ContentNoneException(
+-                '{db_name}:There is no relevant data in the provided \
+-                     binary package '.format(db_name=db_name))
++                "{db_name}:There is no relevant data in the provided "
++                "binary package ".format(db_name=db_name))
+         for index, bin_package_item in enumerate(bin_packaegs):
+             try:
+                 src_package_name = bin_package_item.get('rpm_sourcerpm').split(
+@@ -441,8 +441,8 @@ class InitDataBase():
+         requires_datas = self.__get_data()
+         if requires_datas is None:
+             raise ContentNoneException(
+-                '{db_name}:There is no relevant data in the provided binary \
+-                    dependency package'.format(db_name=db_name))
++                "{db_name}:There is no relevant data in the provided binary "
++                "dependency package".format(db_name=db_name))
+ 
+         with DBHelper(db_name=db_name) as database:
+             database.batch_add(requires_datas, BinRequires)
+@@ -462,8 +462,8 @@ class InitDataBase():
+         provides_datas = self.__get_data()
+         if provides_datas is None:
+             raise ContentNoneException(
+-                '{db_name}:There is no relevant data in the provided \
+-                    binary component '.format(db_name=db_name))
++                "{db_name}:There is no relevant data in the provided "
++                "binary component ".format(db_name=db_name))
+ 
+         with DBHelper(db_name=db_name) as database:
+             database.batch_add(provides_datas, BinProvides)
+@@ -474,8 +474,8 @@ class InitDataBase():
+         files_datas = self.__get_data()
+         if files_datas is None:
+             raise ContentNoneException(
+-                '{db_name}:There is no relevant binary file installation\
+-                    path data in the provided database '.format(db_name=db_name))
++                "{db_name}:There is no relevant binary file installation "
++                "path data in the provided database ".format(db_name=db_name))
+ 
+         with DBHelper(db_name=db_name) as database:
+             database.batch_add(files_datas, BinFiles)
+diff --git a/packageship/libs/dbutils/sqlalchemy_helper.py b/packageship/libs/dbutils/sqlalchemy_helper.py
+index a0b22e2..d18b115 100644
+--- a/packageship/libs/dbutils/sqlalchemy_helper.py
++++ b/packageship/libs/dbutils/sqlalchemy_helper.py
+@@ -279,8 +279,8 @@ class DBHelper(BaseHelper):
+ 
+         if not isinstance(dicts, list):
+             raise TypeError(
+-                'The input for bulk insertion must be a dictionary \
+-                list with the same fields as the current entity')
++                "The input for bulk insertion must be a dictionary"
++                "list with the same fields as the current entity")
+         try:
+             self.session.execute(
+                 model.__table__.insert(),
+diff --git a/packageship/pkgship.py b/packageship/pkgship.py
+index 884b2ab..f9408c8 100644
+--- a/packageship/pkgship.py
++++ b/packageship/pkgship.py
+@@ -25,8 +25,8 @@ try:
+ 
+     LOGGER = Log(__name__)
+ except ImportError as import_error:
+-    print('Error importing related dependencies, \
+-            please check if related dependencies are installed')
++    print("Error importing related dependencies,"
++          "please check if related dependencies are installed")
+ else:
+     from packageship.application.apps.package.function.constants import ResponseCode
+     from packageship.application.apps.package.function.constants import ListNode
+@@ -230,7 +230,9 @@ class PkgshipCommand(BaseCommand):
+                     if package_all.get("not_found_components"):
+                         print("Problem: Not Found Components")
+                         for not_found_com in package_all.get("not_found_components"):
+-                            print("  - nothing provides {} needed by {} ".format(not_found_com, params.packagename))
++                            print(
++                                "  - nothing provides {} needed by {} ".
++                                format(not_found_com, params.packagename))
+                     package_all = package_all.get("build_dict")
+ 
+                 for bin_package, package_depend in package_all.items():
+@@ -835,7 +837,9 @@ class InstallDepCommand(PkgshipCommand):
+                 if package_all.get("not_found_components"):
+                     print("Problem: Not Found Components")
+                     for not_found_com in package_all.get("not_found_components"):
+-                        print("  - nothing provides {} needed by {} ".format(not_found_com, params.packagename))
++                        print(
++                            "  - nothing provides {} needed by {} ".
++                            format(not_found_com, params.packagename))
+                 for bin_package, package_depend in package_all.get("install_dict").items():
+                     # distinguish whether the current data is the data of the root node
+                     if isinstance(package_depend, list) and package_depend[-1][0][0] != 'root':
+@@ -1061,7 +1065,9 @@ class SelfBuildCommand(PkgshipCommand):
+                 if package_all.get("not_found_components"):
+                     print("Problem: Not Found Components")
+                     for not_found_com in package_all.get("not_found_components"):
+-                        print("  - nothing provides {} needed by {} ".format(not_found_com, params.packagename))
++                        print(
++                            "  - nothing provides {} needed by {} ".
++                            format(not_found_com, params.packagename))
+                 bin_package_count = self._parse_bin_package(
+                     package_all.get('binary_dicts'))
+ 

0003-fix-log_level-configuration-item-not-work.patch

@@ -0,0 +1,55 @@
+diff --git a/packageship/application/__init__.py b/packageship/application/__init__.py
+index 1361058..6a57a2e 100644
+--- a/packageship/application/__init__.py
++++ b/packageship/application/__init__.py
+@@ -2,8 +2,6 @@
+ """
+    Initial operation and configuration of the flask project
+ """
+-import sys
+-import threading
+ from flask import Flask
+ from flask_session import Session
+ from flask_apscheduler import APScheduler
+@@ -19,7 +17,9 @@ def _timed_task(app):
+     """
+     Timed task function
+     """
+-    from .apps.lifecycle.function.download_yaml import update_pkg_info  # pylint: disable=import-outside-toplevel
++    # disable=import-outside-toplevel Avoid circular import problems,so import inside the function
++    # pylint: disable=import-outside-toplevel
++    from packageship.application.apps.lifecycle.function.download_yaml import update_pkg_info
+ 
+     _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH)
+     try:
+@@ -34,6 +34,7 @@ def _timed_task(app):
+         if _minute < 0 or _minute > 59:
+             _minute = 0
+ 
++    # disable=no-member Dynamic variable pylint is not recognized
+     app.apscheduler.add_job(  # pylint: disable=no-member
+         func=update_pkg_info, id="update_package_data", trigger="cron", hour=_hour, minute=_minute)
+     app.apscheduler.add_job(  # pylint: disable=no-member
+@@ -52,7 +53,8 @@ def init_app(operation):
+     app = Flask(__name__)
+ 
+     # log configuration
+-    app.logger.addHandler(setup_log(Config))
++    # disable=no-member Dynamic variable pylint is not recognized
++    app.logger.addHandler(setup_log(Config()))  # pylint: disable=no-member
+ 
+     # Load configuration items
+ 
+@@ -66,10 +68,12 @@ def init_app(operation):
+     # Open session function
+     Session(app)
+ 
++    # Variables OPERATION need to be modified within the function and imported in other modules
+     global OPERATION  # pylint: disable=global-statement
+     OPERATION = operation
+ 
+     # Register Blueprint
++    # disable=import-outside-toplevel Avoid circular import problems,so import inside the function
+     from packageship.application import apps  # pylint: disable=import-outside-toplevel
+     for blue, api in apps.blue_point:
+         api.init_app(app)

0004-fix-the-error-when-executing-query-commands.patch

@@ -0,0 +1,24 @@
+diff --git a/packageship/application/apps/package/function/packages.py b/packageship/application/apps/package/function/packages.py
+index eb96087..d36fc34 100644
+--- a/packageship/application/apps/package/function/packages.py
++++ b/packageship/application/apps/package/function/packages.py
+@@ -313,7 +313,8 @@ def _sub_pack(src_name, table_name):
+                 pro_info = res[pro_obj.sub_name]["provides"]
+                 if pro_obj.sub_pro_name in pro_info:
+                     pro_info[pro_obj.sub_pro_name]["requiredby"].update(
+-                        {pro_obj.sub_reqby_name: pro_obj.sub_reqby_name})
++                        {pro_obj.sub_reqby_name: pro_obj.sub_reqby_name}
++                        if pro_obj.sub_reqby_name else {})
+                 else:
+                     pro_info.update(
+                         {
+@@ -368,7 +369,8 @@ def _sub_pack(src_name, table_name):
+                 req_info = sub_pkg_info["requires"]
+                 if req_obj.sub_req_name in req_info:
+                     req_info[req_obj.sub_req_name]["providedby"].update(
+-                        {req_obj.sub_proby_name: req_obj.sub_proby_name})
++                        {req_obj.sub_proby_name: req_obj.sub_proby_name}
++                        if req_obj.sub_proby_name else {})
+                 else:
+                     req_info.update(
+                         {

0005-fix-the-error-when-source-package-has-no-sub-packages.patch

@@ -0,0 +1,62 @@
+diff --git a/packageship/application/apps/package/function/self_depend.py b/packageship/application/apps/package/function/self_depend.py
+index 1ec4c28..b06b950 100644
+--- a/packageship/application/apps/package/function/self_depend.py
++++ b/packageship/application/apps/package/function/self_depend.py
+@@ -106,16 +106,20 @@ class SelfDepend():
+         """
+         if packtype == 'source':
+             response_code, subpack_list = self.search_db.get_sub_pack([packname])
+-            if subpack_list:
+-                for subpack_tuple, dbname in subpack_list:
+-                    self.source_dicts.append_src(packname, dbname, subpack_tuple.search_version)
+-                    if dbname != 'NOT FOUND':
+-                        self.binary_dict.append_bin(key=subpack_tuple.subpack_name,
+-                                                    src=packname,
+-                                                    version=subpack_tuple.search_version,
+-                                                    dbname=dbname)
+-                    else:
+-                        return ResponseCode.PACK_NAME_NOT_FOUND
++            if not subpack_list:
++                return ResponseCode.PACK_NAME_NOT_FOUND
++
++            for subpack_tuple, dbname in subpack_list:
++                self.source_dicts.append_src(packname, dbname, subpack_tuple.search_version)
++                if dbname == 'NOT FOUND':
++                    continue
++
++                if subpack_tuple.subpack_name and subpack_tuple.subpack_name \
++                        not in self.binary_dict.dictionary:
++                    self.binary_dict.append_bin(key=subpack_tuple.subpack_name,
++                                                src=packname,
++                                                version=subpack_tuple.search_version,
++                                                dbname=dbname)
+ 
+         else:
+             response_code, dbname, source_name, version = \
+@@ -178,15 +182,17 @@ class SelfDepend():
+             self.search_subpack_list.remove(None)
+         _, result_list = self.search_db.get_sub_pack(self.search_subpack_list)
+         for subpack_tuple, dbname in result_list:
+-            if dbname != 'NOT FOUND':
+-                if subpack_tuple.subpack_name and subpack_tuple.subpack_name \
+-                        not in self.binary_dict.dictionary:
+-                    self.binary_dict.append_bin(key=subpack_tuple.subpack_name,
+-                                                src=subpack_tuple.search_name,
+-                                                version=subpack_tuple.sub_pack_version,
+-                                                dbname=dbname,
+-                                                parent_node=[subpack_tuple.search_name, 'Subpack'])
+-                    self.search_install_list.append(subpack_tuple.subpack_name)
++            if dbname == 'NOT FOUND':
++                continue
++
++            if subpack_tuple.subpack_name and subpack_tuple.subpack_name \
++                    not in self.binary_dict.dictionary:
++                self.binary_dict.append_bin(key=subpack_tuple.subpack_name,
++                                            src=subpack_tuple.search_name,
++                                            version=subpack_tuple.sub_pack_version,
++                                            dbname=dbname,
++                                            parent_node=[subpack_tuple.search_name, 'Subpack'])
++                self.search_install_list.append(subpack_tuple.subpack_name)
+         self.search_subpack_list.clear()
+ 
+     def query_build(self, selfbuild):

pkgship.spec

@@ -1,6 +1,6 @@
 Name:           pkgship
 Version:        1.1.0
-Release:        4
+Release:        9
 Summary:        Pkgship implements rpm package dependence ,maintainer, patch query and so no.
 License:        Mulan 2.0
 URL:            https://gitee.com/openeuler/openEuler-Advisor
@@ -10,6 +10,23 @@ Source0:        https://gitee.com/openeuler/openEuler-Advisor/pkgship-%{version}
 # extract multiplexing functions, add corresponding docString, and clear pylint
 Patch0:         0001-solve-installation-dependency-query-error.patch
 
+# Fix the problem of continuous spaces in message information in log records
+Patch1:         0002-fix-the-problem-of-continuous-spaces.patch
+
+# When initializing logging, modify the incoming class object to an instance of the class,
+# ensure the execution of internal functions,and read configuration file content
+Patch2:         0003-fix-log_level-configuration-item-not-work.patch
+
+# Fix the error when executing query commands
+Patch3:         0004-fix-the-error-when-executing-query-commands.patch
+
+# Add the judgment of whether the subpack_name attribute exists, fix the code indentation problem, 
+# and reduce the judgment branch of the old code.
+Patch4:         0005-fix-the-error-when-source-package-has-no-sub-packages.patch
+ 
+# Solve the problem of data duplication, increase the maximum queue length judgment, 
+# and avoid occupying too much memory
+Patch5:         0006-fix-data-duplication-issue.patch
 BuildArch:      noarch
 
 BuildRequires: python3-flask-restful python3-flask python3 python3-pyyaml python3-sqlalchemy
@@ -58,8 +75,27 @@ rm -rf $log_path
 %attr(0755,root,root) %{_bindir}/pkgship
 
 %changelog
+* Tue Sep 21 2020 Shenmei Tu <tushenmei@huawei.com> - 1.0-0-9
+- Solve the problem of data duplication, increase the maximum queue length judgment, 
+- and avoid occupying too much memory
+
+* Mon Sep 21 2020 Shenmei Tu <tushenmei@huawei.com> - 1.0-0-8
+- Add the judgment of whether the subpack_name attribute exists, fix the code indentation problem, 
+- and reduce the judgment branch of the old code.
+
+* Mon Sep 21 2020 Shenmei Tu <tushenmei@huawei.com> - 1.0-0-7
+- fix the error when executing query commands
+
+* Mon Sep 21 2020 Shenmei Tu <tushenmei@huawei.com> - 1.0-0-6
+- When initializing logging, modify the incoming class object to an instance of the class,
+- ensure the execution of internal functions,and read configuration file content
+
+* Mon Sep 21 2020 Shenmei Tu <tushenmei@huawei.com> - 1.0-0-5
+- Fix the problem of continuous spaces in message information in log records
+
 * Thu Sep 17 2020 Shenmei Tu <tushenmei@huawei.com> - 1.0-0-4
-- Modify the query logic of package information, reduce redundant queries and align dnf query results, extract multiplexing functions, add corresponding docString, and clear pylint
+- Modify the query logic of package information, reduce redundant queries and align dnf query results, 
+- extract multiplexing functions, add corresponding docString, and clear pylint
 
 * Fri Sep 11 2020 Yiru Wang <wangyiru1@huawei.com> - 1.1.0-3
 - #I1UCM8, #I1UC8G: Modify some config files' permission issue;