Upload files to "/"

2024-08-09 16:45:38 +00:00 · 2024-08-09 16:45:38 +00:00 · b77c64f8d8
commit b77c64f8d8
5 changed files with 207 additions and 0 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -0,0 +1,52 @@
+variables:
+  # placeholders for your actual variables
+  EXTRA_REPO_URL: git@xxx.com
+  EXTRA2_REPO_URL: git@xxx.com
+  NEXUS_IP: "x.x.x.x:8088"
+  NEXUS2_IP: "x.x.x.x:8088"
+  LATEST_BRANCH: "branch_xxx"
+
+stages:
+  - Nexus_Cleanup
+  - Nexus_Vulnerability
+
+before_script:
+  - python3 -m venv venv
+  - source venv/bin/activate
+  - pip install -r cicd_requirements.txt
+  - export EXTRA_REPO_DIR=$(mktemp -d)
+  - export EXTRA2_REPO_DIR=$(mktemp -d)
+
+after_script:
+  - rm -r $EXTRA_REPO_DIR
+  - rm -r $EXTRA2_REPO_DIR
+
+Nexus_Vulnerability:
+  stage: Nexus_Vulnerability
+  needs: [Nexus_Cleanup]
+  tags:
+    - RUNNER-NAME #placeholder for your runner tag
+  script:
+    - echo -e "\e[32mChecking Nexus Pypi for known vulnerabilities.\e[0m"
+    - python3 cicd_parser.py nexus_vulnerability
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: always
+
+
+Nexus_Cleanup:
+  stage: Nexus_Cleanup
+  needs: []
+  tags:
+    - IPC-PLZ
+  script:
+    - echo -e "\e[32mFetching other repos.\e[0m"
+    - echo "Cloning EXTRA repo..."
+    - git clone -q --branch $LATEST_BRANCH "$EXTRA_REPO_URL" "$EXTRA_REPO_DIR"
+    - echo "Cloning EXTRA2 repo..."
+    - git clone -q --branch $LATEST_BRANCH "$EXTRA2_REPO_URL" "$EXTRA2_REPO_DIR"
+    - echo -e "\e[32mNexus Cleanup Started.\e[0m"
+    - python3 cicd_parser.py nexus_cleanup
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: always
--- a/12
+++ b/12
@ -0,0 +1,12 @@
+Proprietary License
+
+This project and the accompanying materials are made available under the terms of this Proprietary License which accompanies this distribution.
+
+NO WARRANTY
+
+ANY USE OF THE PROVIDED SOFTWARE IS AT YOUR OWN RISK. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT ANY WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE, AND NON-INFRINGEMENT.
+
+LIMITATION OF LIABILITY
+
+IN NO EVENT AND UNDER NO LEGAL THEORY, WHETHER IN TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, UNLESS REQUIRED BY APPLICABLE LAW (SUCH AS DELIBERATE AND GROSSLY NEGLIGENT ACTS) OR AGREED TO IN WRITING, SHALL THE AUTHORS BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER ARISING AS A RESULT OF THIS LICENSE OR THE USE OF THE SOFTWARE INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN IF THE AUTHORS HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+
--- a/cicd_parser.py
+++ b/cicd_parser.py
@ -0,0 +1,105 @@
+import argparse
+from module_fetch import Nexus, Project
+import os
+
+
+class bcolors:
+    HEADER = '\033[95m'
+    OKBLUE = '\033[94m'
+    OKCYAN = '\033[96m'
+    OKGREEN = '\033[92m'
+    WARNING = '\033[93m'
+    FAIL = '\033[91m'
+    ENDC = '\033[0m'
+
+
+def main():
+    """
+    Works just as arguments parses.
+    Parsed Arguments:
+        str: nexus_upload
+        str: nexus_vulnerability
+        str: nexus_cleanup
+    """
+    nexus_ips = [os.environ['NEXUS_IP'], os.environ['NEXUS2_IP']]
+    problems = []
+    for ip in nexus_ips:
+        try:
+            parser = argparse.ArgumentParser(description="Run Nexus and Project operations for CI/CD pipeline")
+            parser.add_argument("action", choices=['nexus_upload', 'nexus_vulnerability', 'nexus_cleanup'],
+                                help="Action to perform")
+            args = parser.parse_args()
+            project = Project()
+            nexus = Nexus(ip)
+            if args.action == "nexus_vulnerability":
+                print(f"{bcolors.OKBLUE}DEBUG: {bcolors.ENDC} Running fetch_pypi... for Nexus {ip}")
+                nexus_libs = nexus.fetch_pypi()
+                print(f"{bcolors.OKBLUE}DEBUG: {bcolors.ENDC} Running vulnerability scan... for Nexus {ip}")
+                vulnerabilities = nexus.check_vulnerabilities(packages=nexus_libs)
+                print(f"{bcolors.OKGREEN}DEBUG: {bcolors.ENDC} Finished for Nexus {ip}")
+                if vulnerabilities is not None:
+                    for vuln in vulnerabilities:
+                        print(f"{bcolors.FAIL}Vulnerability found: {bcolors.ENDC} {vuln}")
+                    print(f"{bcolors.FAIL}DEBUG: {bcolors.ENDC} Vulnerabilities found. Exiting for Nexus {ip}")
+                    problems.append(
+                        f"{bcolors.FAIL}DEBUG: {bcolors.ENDC} Vulnerabilities found. Exiting for Nexus {ip}")
+                    continue
+
+            elif args.action == "nexus_upload":
+                print(f"{bcolors.OKBLUE}DEBUG: {bcolors.ENDC} Running fetch_pypi...for Nexus {ip}")
+                nexus_libs = nexus.fetch_pypi()
+                print(f"{bcolors.OKBLUE}DEBUG: {bcolors.ENDC} Fetching libs missing on Nexus {ip}...")
+                missing_libs = project.compare_libs(nexus_libs=nexus_libs, mode='missing')
+                if len(missing_libs) < 1:
+                    print(f"{bcolors.WARNING}DEBUG: {bcolors.ENDC} No missing libs found. Exiting for Nexus {ip}")
+                    continue
+                print(f"{bcolors.OKBLUE}DEBUG: {bcolors.ENDC} Following libs are missing: ")
+                for name, versions in missing_libs.items():
+                    for version in versions:
+                        print(f"Name: {name} | Version: {version}")
+                print(f"{bcolors.OKBLUE}DEBUG: {bcolors.ENDC} Running vulnerability scan...")
+                vulnerabilities = nexus.check_vulnerabilities(packages=missing_libs)
+
+                if vulnerabilities is not None:
+                    for vuln in vulnerabilities:
+                        print(f"{bcolors.FAIL}Vulnerability found: {bcolors.ENDC} {vuln}")
+                    print(f"{bcolors.FAIL}DEBUG: {bcolors.ENDC} Vulnerabilities found. Exiting for Nexus {ip}")
+                    problems.append(
+                        f"{bcolors.FAIL}DEBUG: {bcolors.ENDC} Vulnerabilities found. Exiting for Nexus {ip}")
+                    continue
+                print(f"{bcolors.OKBLUE}DEBUG: {bcolors.ENDC} Downloading WHLS")
+                project.download_whls(missing_libs=missing_libs)
+                print(f"{bcolors.OKBLUE}DEBUG: {bcolors.ENDC} Uploading WHLS to Nexus {ip}")
+                nexus.upload_whls()
+                print(f"{bcolors.OKGREEN}DEBUG: {bcolors.ENDC} Finished for Nexus {ip}")
+
+            elif args.action == "nexus_cleanup":
+                extra_repos = [os.environ['EXTRA_REPO_DIR'], os.environ['EXTRA2_REPO_DIR']]
+                print(f"{bcolors.OKBLUE}DEBUG: {bcolors.ENDC} Running fetch_pypi...for Nexus {ip}")
+                nexus_libs = nexus.fetch_pypi()
+                print(f"{bcolors.OKBLUE}DEBUG: {bcolors.ENDC} Fetching extra libs on Nexus {ip}")
+                extra_libs = project.compare_libs(nexus_libs=nexus_libs, mode='extra', extra_repos=extra_repos)
+                if len(extra_libs) < 1:
+                    print(f"{bcolors.WARNING}DEBUG: {bcolors.ENDC} No extra libs found. Exiting for Nexus {ip}")
+                    continue
+                print(f"{bcolors.OKBLUE}DEBUG: {bcolors.ENDC} Following libs are unused on Nexus {ip} ")
+                for name, versions in extra_libs.items():
+                    for version in versions:
+                        print(f"Name: {name} | Version: {version}")
+
+                print(f"{bcolors.OKBLUE}DEBUG: {bcolors.ENDC} Deleting unused modules from Nexus {ip}")
+                nexus.del_unused(extra_libs=extra_libs)
+                print(f"{bcolors.OKGREEN}DEBUG: {bcolors.ENDC} Finished for Nexus {ip}")
+        except Exception as e:
+            print(f"{bcolors.FAIL}DEBUG: {bcolors.ENDC} Exception for device: {ip}.\n EXCEPTION: {e}")
+            problems.append(
+                f"{bcolors.FAIL}DEBUG: {bcolors.ENDC} Exception for device: {ip}.\n {bcolors.FAIL}EXCEPTION: {bcolors.ENDC}{e}")
+            continue
+    if len(problems) > 0:
+        for problem in problems:
+            print(problem)
+        exit(1)
+
+
+if __name__ == "__main__":
+    main()
--- a/cicd_requirements.txt
+++ b/cicd_requirements.txt
@ -0,0 +1,7 @@
+requests
+packaging
+deepdiff
+requests
+beautifulsoup4
+safety
+twine
--- a/extra_repo.yml
+++ b/extra_repo.yml
@ -0,0 +1,31 @@
+# This would normally be part of the extra repos from where you want to upload Python packages to Nexus.
+# TODO: Move to desired repo and rename to .gitlab-ci.yml
+variables:
+  # placeholders for your actual variables
+  CICD_URL: git@xxx.com
+  NEXUS_IP: "x.x.x.x:8088"
+  NEXUS2_IP: "x.x.x.x:8088"
+  LATEST_BRANCH: "branch_xxx"
+  
+stages:
+  - Nexus_Upload_Libs
+#------------------------------------------------------------
+Nexus_Upload_Libs:
+  stage: Nexus_Upload_Libs
+  needs: []
+  tags:
+    - IPC-PLZ
+  script:
+    - export CICD_DIR=$(mktemp -d)
+    - git clone -q "CICD_URL" "CICD_DIR"
+    - cp -r $CICD_DIR/cicd* .
+    - cp -r $CICD_DIR/module_fetch.py .
+    - rm -r $CICD_DIR
+    - python3 -m venv venv
+    - source venv/bin/activate
+    - pip install -r cicd_requirements.txt
+    - echo -e "\e[32mChecking for missing libs on Nexus.\e[0m"
+    - python3 cicd_parser.py nexus_upload
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && ( $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == LATEST_BRANCH || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" )
+      when: on_success