Add telemetry dataset generator logic

src/telemetry/plugins/module_utils/input_validation/core/config.py

@@ -17,7 +17,12 @@
 Standalone config.py scoped to the three telemetry input files only.
 """
 
-INPUT_VALIDATOR_LOG_PATH = "/var/log/omnia/telemetry"
+import os
+
+INPUT_VALIDATOR_LOG_PATH = os.path.join(
+    os.environ.get("OMNIA_DATA_PATH") or "/opt/omnia",
+    "telemetry", "log"
+)
 
 files = {
     "telemetry_config": "telemetry_config.yml",

src/telemetry/plugins/modules/validate_input.py

@@ -100,12 +100,12 @@
   sample:
     - "Input validation completed for: telemetry input configuration(s)."
     - "Tag(s) run: ['telemetry']. "
-    - "Look at the logs for more details: filename=/var/log/omnia/telemetry/validate_telemetry_input.log"
+    - "Look at the logs for more details: filename=/opt/omnia/telemetry/log/project_default/validate_telemetry_input.log"
 log_file:
   description: Path to the validation log file.
   type: str
   returned: always
-  sample: "/var/log/omnia/telemetry/validate_telemetry_input.log"
+  sample: "/opt/omnia/telemetry/log/project_default/validate_telemetry_input.log"
 errors:
   description: List of error messages from failed validations.
   type: list
@@ -218,7 +218,11 @@ def createlogger(project_name, tag_name=None):
     """
     log_filename = "validate_telemetry_input.log"
 
-    log_file_path = os.path.join(config.INPUT_VALIDATOR_LOG_PATH, log_filename)
+    log_file_path = os.path.join(
+        config.INPUT_VALIDATOR_LOG_PATH,
+        project_name,
+        log_filename
+    )
     os.makedirs(os.path.dirname(log_file_path), exist_ok=True)
     logging.basicConfig(
         filename=log_file_path,
@@ -394,8 +398,11 @@ def main():
 
     logger.error(en_us_validation_msg.get_footer())
 
-    log_file_name = os.path.join(config.INPUT_VALIDATOR_LOG_PATH,
-                                 "validate_telemetry_input.log")
+    log_file_name = os.path.join(
+        config.INPUT_VALIDATOR_LOG_PATH,
+        project_name,
+        "validate_telemetry_input.log"
+    )
 
     status_bool = all(vstatus)
     status_str = "completed" if status_bool else "failed"

test/telemetry/conftest.py

@@ -236,10 +236,21 @@ def _get_order(item):
 # =============================================================================
 
 def _apply_dataset_overrides(config):
-    """Apply dataset/sync overrides from environment variables."""
+    """Apply dataset/sync overrides from environment variables.
+
+    Environment variables (set by run_validation.sh --config mode):
+      OMNIA_DATASET_OVERRIDE      — override config["dataset"]
+      OMNIA_SYNC_INPUT_OVERRIDE   — override config["sync_telemetry_input"]
+
+    Args:
+        config: Test configuration dict from load_test_config().
+
+    Returns:
+        dict: Updated config dict (mutated in place).
+    """
     ds_override = os.environ.get("OMNIA_DATASET_OVERRIDE", "")
     if ds_override:
-        log(f"Dataset override: {config.get('dataset')} -> {ds_override}", "INFO")
+        log(f"Dataset override: {config.get('dataset')} → {ds_override}", "INFO")
         config["dataset"] = ds_override
 
     si_override = os.environ.get("OMNIA_SYNC_INPUT_OVERRIDE", "")
@@ -290,7 +301,7 @@ def pytest_sessionstart(session):
             log(f"Project sync failed: {sync_result['error']}", "WARN")
 
     if config.get("sync_telemetry_input", False):
-        sync_result = sync_telemetry_input(host)
+        sync_result = sync_telemetry_input(host, config)
         if sync_result["success"]:
             log(sync_result["details"], "OK")
         else:

test/telemetry/datasets/README.md

@@ -1,24 +1,143 @@
-# Telemetry Test Datasets
+# Datasets
 
-Test input datasets for the telemetry FVT module.
+Test input configuration for the `telemetry` automation framework.
 
-## Generating Datasets
+## Default Mode (Recommended)
 
-Use the generator tool to create datasets:
+By default (`dataset: ""` in `test_config.yml`), no generated dataset folder is
+needed. The target keeps using its existing runtime files. If sync is enabled,
+the corresponding local sync source comes directly from `src/`:
+
+| File | Source |
+|------|--------|
+| `telemetry_config.yml` | `src/telemetry/input/` |
+| `telemetry_storage_config.yml` | `src/telemetry/input/` |
+| `telemetry_packages.yml` | `src/telemetry/input/` |
+
+With sync disabled, nothing is copied and execution reads the files already
+present on the target under `$OMNIA_DATA_PATH`.
+
+## Custom Datasets
+
+Set `dataset: "my_ds"` in `test_config.yml` to select a custom local sync
+source from `datasets/<name>/`. Generate one with the source-first generator:
+
+```bash
+cd datasets/generator/
+./generate_dataset.py profiles
+./generate_dataset.py profiles defaults
+./generate_dataset.py create my_ds --profile defaults --dry-run
+./generate_dataset.py create my_ds --profile defaults
+```
+
+The ten canonical profiles select supported source combinations and deployment modes:
+
+| Profile | Use Case |
+|---------|----------|
+| `idrac_powerscale` | iDRAC + PowerScale |
+| `idrac_ldms_powerscale` | iDRAC + LDMS + PowerScale |
+| `idrac_ldms` | iDRAC + LDMS (HPC cluster baseline) |
+| `idrac_ome_ufm` | iDRAC + OME + UFM |
+| `idrac_powerscale_vast` | iDRAC + PowerScale + VAST |
+| `ldms_only` | LDMS only (compute-node metrics) |
+| `powerscale_only` | PowerScale metrics and logs only |
+| `defaults` | Canonical source defaults (iDRAC, LDMS, PowerScale, and OME) |
+| `online_mode` | Canonical source defaults with online installation |
+| `offline_mode` | Canonical source defaults with offline installation |
+
+SFM is not configured under `telemetry_sources`. It connects externally to the
+VictoriaMetrics remote-write endpoint exposed by the telemetry deployment.
+
+**Important: Offline mode requires repo_url configuration**
+
+When using the `offline_mode` profile, you **must** configure `repo_url` in the generated `telemetry_packages.yml`:
+
+```bash
+# Generate the dataset
+cd datasets/generator/
+./generate_dataset.py create my_offline --profile offline_mode
+
+# Edit the generated telemetry_packages.yml
+nano ../my_offline/input/telemetry_packages.yml
+```
+
+Set repo_url to your Pulp repository base URL
+
+# Example Format: https://<ip_or_hostname>:<port>/pulp/content/opt/omnia/offline_repo/cluster/<arch>/<os>/<version>
+
+The `repo_url` is used to construct all offline package URLs:
+- Helm charts: `<repo_url>/tarball/<package>/<filename>`
+- Git repos: `<repo_url>/git/<package>/<filename>`
+- Pip modules: `<repo_url>/pip_module/<package>==<version>/`
+
+Leave `repo_url: ""` for `online_mode` (not needed).
+
+For a quick PowerScale-only dataset:
 
 ```bash
-cd generator/
-python generate_dataset.py <name> <profile>
+./generate_dataset.py create my_powerscale --profile powerscale_only
+```
+
+See [`generator/README.md`](generator/README.md) for full usage.
+
+### Custom Dataset Structure
+
+```
+datasets/<name>/
+  input/
+    telemetry_config.yml
+    telemetry_storage_config.yml
+    telemetry_packages.yml
+  dataset_manifest.yml
+  README.md
 ```
 
-See `generator/README.md` for full usage.
+Datasets never contain credentials. Appliance credentials (OME, SFM, UFM,
+VAST) are stored in `test_creds.yml` and auto-encrypted with Ansible Vault.
 
-## Dataset Structure
+## Switching Datasets
 
+### Method 1: Edit `test_config.yml` (persistent)
+```yaml
+dataset: "my_custom_ds"    # Use custom dataset
+dataset: ""                # Use src/ only as an optional sync source
+sync_telemetry_input: true # Enable sync to push files to target
 ```
-data_set_01/
-└── input/
-    ├── telemetry_config.yml
-    ├── telemetry_storage_config.yml
-    └── telemetry_packages.yml
+
+Then run normally:
+```bash
+./run_validation.sh fvt_telemetry deploy verify
 ```
+
+### Method 2: Environment variables (one-off)
+Override dataset and/or sync settings without editing the config file:
+
+```bash
+# Override dataset only (sync respects test_config.yml setting)
+OMNIA_DATASET_OVERRIDE=my_custom_ds \
+  ./run_validation.sh fvt_telemetry deploy verify
+
+# Override both dataset and sync
+OMNIA_DATASET_OVERRIDE=my_custom_ds \
+OMNIA_SYNC_INPUT_OVERRIDE=true \
+  ./run_validation.sh fvt_telemetry deploy verify
+
+# Override sync only (dataset from test_config.yml)
+OMNIA_SYNC_INPUT_OVERRIDE=true \
+  ./run_validation.sh fvt_telemetry deploy verify
+```
+
+**Priority:**
+- `OMNIA_DATASET_OVERRIDE` takes priority over `test_config.yml` dataset
+- `OMNIA_SYNC_INPUT_OVERRIDE` takes priority over `test_config.yml` sync_telemetry_input
+
+## Sync Behavior
+
+| Setting | What gets synced |
+|---------|------------------|
+| `sync_telemetry_input: true`, named dataset | `datasets/<name>/input/` → target input path |
+| `sync_telemetry_input: true`, empty dataset | `src/telemetry/input/` → target input path |
+| `sync_telemetry_input: false` | Nothing is synced; target keeps existing files |
+
+The framework reads `OMNIA_DATA_PATH` and `OMNIA_PROJECT_NAME` from the
+target server to resolve sync destinations.