From a59c3917d1b1af6c78271e3819a2b719cfaa6f32 Mon Sep 17 00:00:00 2001 From: Raunak Jalan Date: Sun, 25 Aug 2024 13:53:30 +0530 Subject: [PATCH 01/15] Adding failure reruns and retry for testcases --- .github/workflows/e2e.yaml | 28 +++++++++++--- e2e/e2e.py | 75 +++++++++++++++++++++----------------- 2 files changed, 63 insertions(+), 40 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 3d57a507b..f8b0e9125 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -21,6 +21,17 @@ on: description: 'Name of test to run. Empty to run all' required: false default: '' + run_failed_tests_only: + description: "Run only the failed tests from the last run" + required: false + default: false + type: boolean + retry_count: + description: "Number of retries for failed test cases" + required: false + default: 1 + type: number + jobs: e2e: runs-on: self-hosted @@ -83,8 +94,10 @@ jobs: cd $GITHUB_WORKSPACE/simplyBlockDeploy terraform plan \ -var "mgmt_nodes=1" -var "storage_nodes=3" -var "volumes_per_storage_nodes=3" \ - -var "extra_nodes=1" -var "extra_nodes_instance_type=m6id.large" \ - -var "region=us-east-2" -var "sbcli_cmd=${{ github.event.inputs.sbcli_cmd || 'sbcli-dev' }}" -out=tfplan + -var mgmt_nodes_instance_type="m6id.xlarge" -var storage_nodes_instance_type="m6id.xlarge" \ + -var "extra_nodes=1" -var "extra_nodes_instance_type=m6id.xlarge" \ + -var storage_nodes_ebs_size2=100 -var "region=us-east-2" \ + -var "sbcli_cmd=${{ github.event.inputs.sbcli_cmd || 'sbcli-dev' }}" -out=tfplan - name: Apply Terraform Changes run: | cd $GITHUB_WORKSPACE/simplyBlockDeploy @@ -100,13 +113,13 @@ jobs: - name: Bootstrap Cluster run: | cd $GITHUB_WORKSPACE/simplyBlockDeploy - ./bootstrap-cluster.sh --max-lvol 10 --max-snap 10 --max-prov 450g --number-of-devices 3 + ./bootstrap-cluster.sh --max-lvol 100 --max-snap 100 --max-prov 360G --number-of-devices 3 id: bootstrap_cluster env: SBCLI_CMD: ${{ github.event.inputs.sbcli_cmd || 'sbcli-dev' }} - name: Setup Tests & Run Tests - timeout-minutes: 60 + timeout-minutes: 120 run: | cd $GITHUB_WORKSPACE/e2e sudo apt-get install -y python3.12-venv @@ -128,7 +141,10 @@ jobs: if [ -n "${{ github.event.inputs.testname }}" ]; then TESTNAME="--testname ${{ github.event.inputs.testname }}" fi - python3 e2e.py $TESTNAME + if [ "${{ github.event.inputs.run_failed_tests_only }}" = "true" ]; then + python3 e2e.py --failed-only --retry ${{ github.event.inputs.retry_count }} $TESTNAME + else + python3 e2e.py --retry ${{ github.event.inputs.retry_count }} $TESTNAME - name: Upload automation and docker logs to s3 run: | cd $GITHUB_WORKSPACE/e2e/logs @@ -243,4 +259,4 @@ jobs: ls -la ./ rm -rf ./* || true rm -rf ./.??* || true - ls -la ./ \ No newline at end of file + ls -la ./ diff --git a/e2e/e2e.py b/e2e/e2e.py index b224469c2..375a4e68e 100644 --- a/e2e/e2e.py +++ b/e2e/e2e.py @@ -1,6 +1,8 @@ ### simplyblock e2e tests import argparse import traceback +import os +import json from __init__ import get_all_tests from logger_config import setup_logger from exceptions.custom_exception import ( @@ -11,73 +13,78 @@ from utils.sbcli_utils import SbcliUtils from utils.ssh_utils import SshUtils - def main(): - """Run complete test suite - """ + """Run complete test suite or only failed tests from last run.""" parser = argparse.ArgumentParser(description="Run simplyBlock's E2E Test Framework") parser.add_argument('--testname', type=str, help="The name of the test to run", default=None) parser.add_argument('--fio_debug', type=bool, help="Add debug flag to fio", default=False) + parser.add_argument('--failed-only', action='store_true', help="Run only failed tests from last run", default=False) + parser.add_argument('--retry', type=int, help="Number of retries for failed cases", default=1) args = parser.parse_args() tests = get_all_tests() - # Find the test class based on the provided test name - test_class_run = [] - if args.testname is None or len(args.testname.strip()) == 0: - test_class_run = tests + + # Load previously failed cases if '--failed-only' is set + failed_cases_file = 'failed_cases.json' + if args.failed_only and os.path.exists(failed_cases_file): + with open(failed_cases_file, 'r') as file: + failed_tests = json.load(file) + test_class_run = [cls for cls in tests if cls.__name__ in failed_tests] else: - for cls in tests: - if args.testname.lower() in cls.__name__.lower(): - test_class_run.append(cls) + # Run all tests or selected ones + test_class_run = [] + if args.testname is None or len(args.testname.strip()) == 0: + test_class_run = tests + else: + for cls in tests: + if args.testname.lower() in cls.__name__.lower(): + test_class_run.append(cls) if not test_class_run: available_tests = ', '.join(cls.__name__ for cls in tests) logger.info(f"Test '{args.testname}' not found. Available tests are: {available_tests}") raise TestNotFoundException(args.testname, available_tests) - + errors = {} for test in test_class_run: logger.info(f"Running Test {test}") test_obj = test(fio_debug=args.fio_debug) - try: - test_obj.setup() - test_obj.run() - except Exception as exp: - logger.error(traceback.format_exc()) - errors[f"{test.__name__}"] = [exp] + + # Retry logic + for attempt in range(args.retry): + try: + test_obj.setup() + test_obj.run() + logger.info(f"Test {test.__name__} passed on attempt {attempt + 1}") + break # Test passed, no need for more retries + except Exception as exp: + logger.error(f"Attempt {attempt + 1} failed for test {test.__name__}") + logger.error(traceback.format_exc()) + errors[f"{test.__name__}"] = [exp] + try: test_obj.teardown() - # pass except Exception as _: logger.error(f"Error During Teardown for test: {test.__name__}") logger.error(traceback.format_exc()) - # errors[f"{test.__name__}"].append(exp) - finally: - if check_for_dumps(): - logger.info("Found a core dump during test execution. " - "Cannot execute more tests as cluster is not stable. Exiting") - break failed_cases = list(errors.keys()) + + # Save failed cases for next run + if failed_cases: + with open(failed_cases_file, 'w', encoding='utf-8') as file: + json.dump(failed_cases, file) + logger.info(f"Number of Total Cases: {len(test_class_run)}") logger.info(f"Number of Passed Cases: {len(test_class_run) - len(failed_cases)}") logger.info(f"Number of Failed Cases: {len(failed_cases)}") - - logger.info("Test Wise run status:") - for test in test_class_run: - if test.__name__ not in failed_cases: - logger.info(f"{test.__name__} PASSED CASE.") - else: - logger.info(f"{test.__name__} FAILED CASE.") - if errors: raise MultipleExceptions(errors) - def check_for_dumps(): - """Validates whether core dumps present on machines + """Validates whether core dumps are present on machines Returns: bool: If there are core dumps or not From 40ab31e98c995f4484470fdd666837e8c5b95827 Mon Sep 17 00:00:00 2001 From: Raunak Jalan Date: Sun, 25 Aug 2024 14:38:20 +0530 Subject: [PATCH 02/15] Testing case with failure --- e2e/e2e_tests/single_node_failure.py | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e/e2e_tests/single_node_failure.py b/e2e/e2e_tests/single_node_failure.py index 9f0394935..6834d3e63 100644 --- a/e2e/e2e_tests/single_node_failure.py +++ b/e2e/e2e_tests/single_node_failure.py @@ -65,6 +65,7 @@ def run(self): distr_ndcs=2, distr_npcs=1 ) + raise Exception("Testing failure runs and retry runs") lvols = self.sbcli_utils.list_lvols() assert self.lvol_name in list(lvols.keys()), \ f"Lvol {self.lvol_name} not present in list of lvols post add: {lvols}" From 2d7cbd3ee0b571cc6df3d34c094430f5d07dbada Mon Sep 17 00:00:00 2001 From: Raunak Jalan Date: Sun, 25 Aug 2024 14:43:23 +0530 Subject: [PATCH 03/15] Testing case with failure --- e2e/e2e.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/e2e/e2e.py b/e2e/e2e.py index 375a4e68e..5b09c43e9 100644 --- a/e2e/e2e.py +++ b/e2e/e2e.py @@ -57,6 +57,8 @@ def main(): test_obj.setup() test_obj.run() logger.info(f"Test {test.__name__} passed on attempt {attempt + 1}") + if test.__name__ in errors: + del errors[test.__name__] break # Test passed, no need for more retries except Exception as exp: logger.error(f"Attempt {attempt + 1} failed for test {test.__name__}") @@ -68,6 +70,11 @@ def main(): except Exception as _: logger.error(f"Error During Teardown for test: {test.__name__}") logger.error(traceback.format_exc()) + finally: + if check_for_dumps(): + logger.info("Found a core dump during test execution. " + "Cannot execute more tests as cluster is not stable. Exiting") + break failed_cases = list(errors.keys()) From 3dae1681a4cdd000f6f6c9b1f5e2ef3dad40531d Mon Sep 17 00:00:00 2001 From: Raunak Jalan Date: Sun, 25 Aug 2024 21:23:41 +0530 Subject: [PATCH 04/15] Testing case with failure --- .github/workflows/e2e.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index f8b0e9125..be9427cfd 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -145,6 +145,7 @@ jobs: python3 e2e.py --failed-only --retry ${{ github.event.inputs.retry_count }} $TESTNAME else python3 e2e.py --retry ${{ github.event.inputs.retry_count }} $TESTNAME + fi - name: Upload automation and docker logs to s3 run: | cd $GITHUB_WORKSPACE/e2e/logs From 1a629463b4b1daf3a6b97939487c30075522299a Mon Sep 17 00:00:00 2001 From: Raunak Jalan Date: Sun, 25 Aug 2024 22:05:36 +0530 Subject: [PATCH 05/15] Add check for failed in a branch vs unececuted cases --- .github/workflows/e2e.yaml | 19 ++++++++++++--- e2e/e2e.py | 36 +++++++++++++++++++--------- e2e/e2e_tests/single_node_failure.py | 2 +- 3 files changed, 42 insertions(+), 15 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index be9427cfd..18684417a 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -31,6 +31,11 @@ on: required: false default: 1 type: number + run_unexecuted_tests: + description: "Run tests that were not executed in the last run" + required: false + default: false + type: boolean jobs: e2e: @@ -81,7 +86,7 @@ jobs: -backend-config="dynamodb_table=${TFSTATE_DYNAMODB_TABLE}" \ -backend-config="encrypt=true" - - name: select or create workspace + - name: Select or Create Workspace run: | cd $GITHUB_WORKSPACE/simplyBlockDeploy terraform workspace select -or-create ghiaction-sbclie2e @@ -127,6 +132,12 @@ jobs: source myenv/bin/activate python3 -m pip install -r requirements.txt echo "Running tests in namespace ${{ steps.get-namespace.outputs.namespace }}" + + # Clean branch name to remove slashes for filename + BRANCH_NAME_SAFE=$(echo "${{ github.ref_name }}" | tr '/' '_') + export BRANCH_NAME=${BRANCH_NAME_SAFE} + export FAILED_CASES_FILE="failed_cases_${BRANCH_NAME_SAFE}.json" + export EXECUTED_CASES_FILE="executed_cases_${BRANCH_NAME_SAFE}.json" export CLUSTER_ID=${{ steps.bootstrap_cluster.outputs.cluster_id }} export CLUSTER_SECRET=${{ steps.bootstrap_cluster.outputs.cluster_secret }} export CLUSTER_IP=${{ steps.bootstrap_cluster.outputs.cluster_ip }} @@ -142,9 +153,11 @@ jobs: TESTNAME="--testname ${{ github.event.inputs.testname }}" fi if [ "${{ github.event.inputs.run_failed_tests_only }}" = "true" ]; then - python3 e2e.py --failed-only --retry ${{ github.event.inputs.retry_count }} $TESTNAME + python3 e2e.py --failed-only --retry ${{ github.event.inputs.retry_count }} --branch $BRANCH_NAME $TESTNAME + elif [ "${{ github.event.inputs.run_unexecuted_tests }}" = "true" ]; then + python3 e2e.py --unexecuted-only --retry ${{ github.event.inputs.retry_count }} --branch $BRANCH_NAME $TESTNAME else - python3 e2e.py --retry ${{ github.event.inputs.retry_count }} $TESTNAME + python3 e2e.py --retry ${{ github.event.inputs.retry_count }} --branch $BRANCH_NAME $TESTNAME fi - name: Upload automation and docker logs to s3 run: | diff --git a/e2e/e2e.py b/e2e/e2e.py index 5b09c43e9..f872522c7 100644 --- a/e2e/e2e.py +++ b/e2e/e2e.py @@ -1,36 +1,43 @@ ### simplyblock e2e tests import argparse -import traceback import os import json +import traceback from __init__ import get_all_tests from logger_config import setup_logger -from exceptions.custom_exception import ( - TestNotFoundException, - MultipleExceptions -) +from exceptions.custom_exception import TestNotFoundException, MultipleExceptions from e2e_tests.cluster_test_base import TestClusterBase from utils.sbcli_utils import SbcliUtils from utils.ssh_utils import SshUtils + def main(): - """Run complete test suite or only failed tests from last run.""" + """Run the complete test suite or specific tests.""" parser = argparse.ArgumentParser(description="Run simplyBlock's E2E Test Framework") parser.add_argument('--testname', type=str, help="The name of the test to run", default=None) parser.add_argument('--fio_debug', type=bool, help="Add debug flag to fio", default=False) parser.add_argument('--failed-only', action='store_true', help="Run only failed tests from last run", default=False) + parser.add_argument('--unexecuted-only', action='store_true', help="Run only unexecuted tests from last run", default=False) + parser.add_argument('--branch', type=str, help="Branch name to uniquely store test results", required=True) parser.add_argument('--retry', type=int, help="Number of retries for failed cases", default=1) args = parser.parse_args() tests = get_all_tests() + # File to store failed test cases for the specific branch + failed_cases_file = f'failed_cases_{args.branch}.json' + executed_cases_file = f'executed_cases_{args.branch}.json' + # Load previously failed cases if '--failed-only' is set - failed_cases_file = 'failed_cases.json' if args.failed_only and os.path.exists(failed_cases_file): with open(failed_cases_file, 'r') as file: failed_tests = json.load(file) test_class_run = [cls for cls in tests if cls.__name__ in failed_tests] + elif args.unexecuted_only and os.path.exists(executed_cases_file): + with open(executed_cases_file, 'r') as file: + executed_tests = json.load(file) + test_class_run = [cls for cls in tests if cls.__name__ not in executed_tests] else: # Run all tests or selected ones test_class_run = [] @@ -47,18 +54,17 @@ def main(): raise TestNotFoundException(args.testname, available_tests) errors = {} + executed_tests = [] for test in test_class_run: logger.info(f"Running Test {test}") test_obj = test(fio_debug=args.fio_debug) - # Retry logic for attempt in range(args.retry): try: test_obj.setup() test_obj.run() + executed_tests.append(test.__name__) logger.info(f"Test {test.__name__} passed on attempt {attempt + 1}") - if test.__name__ in errors: - del errors[test.__name__] break # Test passed, no need for more retries except Exception as exp: logger.error(f"Attempt {attempt + 1} failed for test {test.__name__}") @@ -80,8 +86,15 @@ def main(): # Save failed cases for next run if failed_cases: - with open(failed_cases_file, 'w', encoding='utf-8') as file: + with open(failed_cases_file, 'w') as file: json.dump(failed_cases, file) + else: + if os.path.exists(failed_cases_file): + os.remove(failed_cases_file) # Clear file if all tests passed + + # Save executed cases for next run + with open(executed_cases_file, 'w') as file: + json.dump(executed_tests, file) logger.info(f"Number of Total Cases: {len(test_class_run)}") logger.info(f"Number of Passed Cases: {len(test_class_run) - len(failed_cases)}") @@ -90,6 +103,7 @@ def main(): if errors: raise MultipleExceptions(errors) + def check_for_dumps(): """Validates whether core dumps are present on machines diff --git a/e2e/e2e_tests/single_node_failure.py b/e2e/e2e_tests/single_node_failure.py index 6834d3e63..9ca5b1aa7 100644 --- a/e2e/e2e_tests/single_node_failure.py +++ b/e2e/e2e_tests/single_node_failure.py @@ -65,7 +65,7 @@ def run(self): distr_ndcs=2, distr_npcs=1 ) - raise Exception("Testing failure runs and retry runs") + # raise Exception("Testing failure runs and retry runs") lvols = self.sbcli_utils.list_lvols() assert self.lvol_name in list(lvols.keys()), \ f"Lvol {self.lvol_name} not present in list of lvols post add: {lvols}" From 83c161e5f9631cd1b4691eb49f284623a6ae24b8 Mon Sep 17 00:00:00 2001 From: Raunak Jalan Date: Sun, 25 Aug 2024 22:06:28 +0530 Subject: [PATCH 06/15] Add check for failed in a branch vs unececuted cases --- .github/workflows/e2e.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 18684417a..d7f1541a0 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -26,16 +26,16 @@ on: required: false default: false type: boolean - retry_count: - description: "Number of retries for failed test cases" - required: false - default: 1 - type: number run_unexecuted_tests: description: "Run tests that were not executed in the last run" required: false default: false type: boolean + retry_count: + description: "Number of retries for failed test cases" + required: false + default: 1 + type: number jobs: e2e: From 6fd1edd631ffcea8b34fed74c107a69a584baabc Mon Sep 17 00:00:00 2001 From: Raunak Jalan Date: Sun, 25 Aug 2024 23:34:02 +0530 Subject: [PATCH 07/15] Failure test check list --- e2e/e2e.py | 1 + e2e/e2e_tests/single_node_failure.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/e2e/e2e.py b/e2e/e2e.py index f872522c7..5c6a6372e 100644 --- a/e2e/e2e.py +++ b/e2e/e2e.py @@ -48,6 +48,7 @@ def main(): if args.testname.lower() in cls.__name__.lower(): test_class_run.append(cls) + logger.info(f"List of tests to run: {test_class_run}") if not test_class_run: available_tests = ', '.join(cls.__name__ for cls in tests) logger.info(f"Test '{args.testname}' not found. Available tests are: {available_tests}") diff --git a/e2e/e2e_tests/single_node_failure.py b/e2e/e2e_tests/single_node_failure.py index 9ca5b1aa7..6834d3e63 100644 --- a/e2e/e2e_tests/single_node_failure.py +++ b/e2e/e2e_tests/single_node_failure.py @@ -65,7 +65,7 @@ def run(self): distr_ndcs=2, distr_npcs=1 ) - # raise Exception("Testing failure runs and retry runs") + raise Exception("Testing failure runs and retry runs") lvols = self.sbcli_utils.list_lvols() assert self.lvol_name in list(lvols.keys()), \ f"Lvol {self.lvol_name} not present in list of lvols post add: {lvols}" From fe57803d737603dae1e9d71a084f4809a69cc294 Mon Sep 17 00:00:00 2001 From: Raunak Jalan Date: Mon, 26 Aug 2024 00:40:53 +0530 Subject: [PATCH 08/15] Failure test check list --- e2e/e2e.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/e2e/e2e.py b/e2e/e2e.py index 5c6a6372e..a61624155 100644 --- a/e2e/e2e.py +++ b/e2e/e2e.py @@ -28,18 +28,27 @@ def main(): # File to store failed test cases for the specific branch failed_cases_file = f'failed_cases_{args.branch}.json' executed_cases_file = f'executed_cases_{args.branch}.json' + test_classes_total = [cls.__name__ for cls in tests] # Load previously failed cases if '--failed-only' is set if args.failed_only and os.path.exists(failed_cases_file): - with open(failed_cases_file, 'r') as file: + logger.info("Running failed cases only") + with open(failed_cases_file, 'r', encoding='utf-8') as file: failed_tests = json.load(file) - test_class_run = [cls for cls in tests if cls.__name__ in failed_tests] + test_class_run = [cls for cls in tests + if any(ft in f'{cls.__name__}' for ft in failed_tests)] + + logger.info(f"Running failed cases only: {test_class_run}") elif args.unexecuted_only and os.path.exists(executed_cases_file): - with open(executed_cases_file, 'r') as file: + logger.info("Running unexecuted cases only") + with open(executed_cases_file, 'r', encoding='utf-8') as file: executed_tests = json.load(file) - test_class_run = [cls for cls in tests if cls.__name__ not in executed_tests] + test_class_run = [cls for cls in tests + if all(unet not in f'{cls.__name__}' for unet in failed_tests)] + logger.info(f"Running unexecuted cases only: {test_class_run}") else: # Run all tests or selected ones + logger.info("Running all or selected cases") test_class_run = [] if args.testname is None or len(args.testname.strip()) == 0: test_class_run = tests @@ -63,9 +72,11 @@ def main(): for attempt in range(args.retry): try: test_obj.setup() - test_obj.run() executed_tests.append(test.__name__) + test_obj.run() logger.info(f"Test {test.__name__} passed on attempt {attempt + 1}") + if f"{test.__name__}" in errors: + del errors[f"{test.__name__}"] break # Test passed, no need for more retries except Exception as exp: logger.error(f"Attempt {attempt + 1} failed for test {test.__name__}") From c259ae6d84e0e77dd94b3efc7499df8beefb290d Mon Sep 17 00:00:00 2001 From: Raunak Jalan Date: Mon, 26 Aug 2024 01:38:33 +0530 Subject: [PATCH 09/15] Failure test check list --- .github/workflows/e2e.yaml | 4 ++-- e2e/e2e.py | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index d7f1541a0..9ac7fc923 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -153,9 +153,9 @@ jobs: TESTNAME="--testname ${{ github.event.inputs.testname }}" fi if [ "${{ github.event.inputs.run_failed_tests_only }}" = "true" ]; then - python3 e2e.py --failed-only --retry ${{ github.event.inputs.retry_count }} --branch $BRANCH_NAME $TESTNAME + python3 e2e.py --failed_only --retry ${{ github.event.inputs.retry_count }} --branch $BRANCH_NAME $TESTNAME elif [ "${{ github.event.inputs.run_unexecuted_tests }}" = "true" ]; then - python3 e2e.py --unexecuted-only --retry ${{ github.event.inputs.retry_count }} --branch $BRANCH_NAME $TESTNAME + python3 e2e.py --unexecuted_only --retry ${{ github.event.inputs.retry_count }} --branch $BRANCH_NAME $TESTNAME else python3 e2e.py --retry ${{ github.event.inputs.retry_count }} --branch $BRANCH_NAME $TESTNAME fi diff --git a/e2e/e2e.py b/e2e/e2e.py index a61624155..e496ccf94 100644 --- a/e2e/e2e.py +++ b/e2e/e2e.py @@ -16,8 +16,8 @@ def main(): parser = argparse.ArgumentParser(description="Run simplyBlock's E2E Test Framework") parser.add_argument('--testname', type=str, help="The name of the test to run", default=None) parser.add_argument('--fio_debug', type=bool, help="Add debug flag to fio", default=False) - parser.add_argument('--failed-only', action='store_true', help="Run only failed tests from last run", default=False) - parser.add_argument('--unexecuted-only', action='store_true', help="Run only unexecuted tests from last run", default=False) + parser.add_argument('--failed_only', action='store_true', help="Run only failed tests from last run", default=False) + parser.add_argument('--unexecuted_only', action='store_true', help="Run only unexecuted tests from last run", default=False) parser.add_argument('--branch', type=str, help="Branch name to uniquely store test results", required=True) parser.add_argument('--retry', type=int, help="Number of retries for failed cases", default=1) @@ -30,7 +30,11 @@ def main(): executed_cases_file = f'executed_cases_{args.branch}.json' test_classes_total = [cls.__name__ for cls in tests] - # Load previously failed cases if '--failed-only' is set + logger.info(f"Failed only: {args.failed_only}") + logger.info(f"Unexecuted only: {args.unexecuted_only}") + + + # Load previously failed cases if '--failed_only' is set if args.failed_only and os.path.exists(failed_cases_file): logger.info("Running failed cases only") with open(failed_cases_file, 'r', encoding='utf-8') as file: From 2f72818d50e57b7b07a78b51cbb1e957d6a32e25 Mon Sep 17 00:00:00 2001 From: Raunak Jalan Date: Mon, 26 Aug 2024 01:54:43 +0530 Subject: [PATCH 10/15] Failure test check list --- e2e/e2e.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/e2e/e2e.py b/e2e/e2e.py index e496ccf94..2ed80a930 100644 --- a/e2e/e2e.py +++ b/e2e/e2e.py @@ -32,7 +32,8 @@ def main(): logger.info(f"Failed only: {args.failed_only}") logger.info(f"Unexecuted only: {args.unexecuted_only}") - + logger.info(f"Failed case file: {failed_cases_file}") + logger.info(f"File exists: {os.path.exists(failed_cases_file)}") # Load previously failed cases if '--failed_only' is set if args.failed_only and os.path.exists(failed_cases_file): From 4cc4af93b51e1bcf84cc81041473128e39c0f888 Mon Sep 17 00:00:00 2001 From: Raunak Jalan Date: Mon, 26 Aug 2024 01:57:02 +0530 Subject: [PATCH 11/15] Failure test check list --- e2e/e2e.py | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e/e2e.py b/e2e/e2e.py index 2ed80a930..67f86f3c2 100644 --- a/e2e/e2e.py +++ b/e2e/e2e.py @@ -34,6 +34,7 @@ def main(): logger.info(f"Unexecuted only: {args.unexecuted_only}") logger.info(f"Failed case file: {failed_cases_file}") logger.info(f"File exists: {os.path.exists(failed_cases_file)}") + logger.info(f"Current dir: {os.getcwd())}") # Load previously failed cases if '--failed_only' is set if args.failed_only and os.path.exists(failed_cases_file): From 9cf57c16e224da55e40197a3c7348183491c0ee2 Mon Sep 17 00:00:00 2001 From: Raunak Jalan Date: Mon, 26 Aug 2024 02:40:15 +0530 Subject: [PATCH 12/15] Failure test check list --- e2e/e2e.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/e2e.py b/e2e/e2e.py index 67f86f3c2..66c8a3b50 100644 --- a/e2e/e2e.py +++ b/e2e/e2e.py @@ -34,7 +34,7 @@ def main(): logger.info(f"Unexecuted only: {args.unexecuted_only}") logger.info(f"Failed case file: {failed_cases_file}") logger.info(f"File exists: {os.path.exists(failed_cases_file)}") - logger.info(f"Current dir: {os.getcwd())}") + logger.info(f"Current dir: {os.getcwd()}") # Load previously failed cases if '--failed_only' is set if args.failed_only and os.path.exists(failed_cases_file): From 19deb941841a5f653c207ea6c589f8bd0f6f61a7 Mon Sep 17 00:00:00 2001 From: Raunak Jalan Date: Mon, 26 Aug 2024 12:14:49 +0530 Subject: [PATCH 13/15] Failure test check list --- e2e/e2e.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/e2e/e2e.py b/e2e/e2e.py index 66c8a3b50..01f653701 100644 --- a/e2e/e2e.py +++ b/e2e/e2e.py @@ -26,15 +26,21 @@ def main(): tests = get_all_tests() # File to store failed test cases for the specific branch - failed_cases_file = f'failed_cases_{args.branch}.json' - executed_cases_file = f'executed_cases_{args.branch}.json' - test_classes_total = [cls.__name__ for cls in tests] + base_dir = os.path.join(os.path.expanduser('~'), 'e2e_test_runs_fail_unexec_json') + if not os.path.exists(base_dir): + os.makedirs(base_dir) + failed_cases_file = os.path.join(base_dir, + f'failed_cases_{args.branch}.json') + executed_cases_file = os.path.join(base_dir, + f'executed_cases_{args.branch}.json') logger.info(f"Failed only: {args.failed_only}") logger.info(f"Unexecuted only: {args.unexecuted_only}") logger.info(f"Failed case file: {failed_cases_file}") logger.info(f"File exists: {os.path.exists(failed_cases_file)}") - logger.info(f"Current dir: {os.getcwd()}") + + onlyfiles = [f for f in os.listdir(base_dir) if os.path.isfile(os.path.join(base_dir, f))] + logger.info(f"List of files: {onlyfiles}") # Load previously failed cases if '--failed_only' is set if args.failed_only and os.path.exists(failed_cases_file): From 94ed7d7c936335a6b327d1845cdb5451cf1b82c2 Mon Sep 17 00:00:00 2001 From: Raunak Jalan Date: Mon, 26 Aug 2024 15:00:38 +0530 Subject: [PATCH 14/15] Fixing test --- e2e/e2e.py | 2 +- e2e/e2e_tests/single_node_failure.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/e2e.py b/e2e/e2e.py index 01f653701..88dd89890 100644 --- a/e2e/e2e.py +++ b/e2e/e2e.py @@ -56,7 +56,7 @@ def main(): with open(executed_cases_file, 'r', encoding='utf-8') as file: executed_tests = json.load(file) test_class_run = [cls for cls in tests - if all(unet not in f'{cls.__name__}' for unet in failed_tests)] + if all(unet not in f'{cls.__name__}' for unet in executed_tests)] logger.info(f"Running unexecuted cases only: {test_class_run}") else: # Run all tests or selected ones diff --git a/e2e/e2e_tests/single_node_failure.py b/e2e/e2e_tests/single_node_failure.py index 6834d3e63..9ca5b1aa7 100644 --- a/e2e/e2e_tests/single_node_failure.py +++ b/e2e/e2e_tests/single_node_failure.py @@ -65,7 +65,7 @@ def run(self): distr_ndcs=2, distr_npcs=1 ) - raise Exception("Testing failure runs and retry runs") + # raise Exception("Testing failure runs and retry runs") lvols = self.sbcli_utils.list_lvols() assert self.lvol_name in list(lvols.keys()), \ f"Lvol {self.lvol_name} not present in list of lvols post add: {lvols}" From c1a175b1087a7ec6756cd0770dc206ce07924f9d Mon Sep 17 00:00:00 2001 From: Raunak Jalan Date: Mon, 26 Aug 2024 15:36:32 +0530 Subject: [PATCH 15/15] Removing execution file if no tests ran --- e2e/e2e.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/e2e/e2e.py b/e2e/e2e.py index 88dd89890..fde2ae966 100644 --- a/e2e/e2e.py +++ b/e2e/e2e.py @@ -117,8 +117,12 @@ def main(): os.remove(failed_cases_file) # Clear file if all tests passed # Save executed cases for next run - with open(executed_cases_file, 'w') as file: - json.dump(executed_tests, file) + if executed_tests: + with open(executed_cases_file, 'w') as file: + json.dump(executed_tests, file) + else: + if os.path.exists(executed_cases_file): + os.remove(executed_cases_file) # Clear file if no tests executed the run logger.info(f"Number of Total Cases: {len(test_class_run)}") logger.info(f"Number of Passed Cases: {len(test_class_run) - len(failed_cases)}")