diff --git a/experimental/dbx2dab/.gitignore b/experimental/dbx2dab/.gitignore new file mode 100644 index 00000000..576f4fab --- /dev/null +++ b/experimental/dbx2dab/.gitignore @@ -0,0 +1,3 @@ +.venv +__pycache__ +.vscode/settings.json diff --git a/experimental/dbx2dab/.python-version b/experimental/dbx2dab/.python-version new file mode 100644 index 00000000..2c073331 --- /dev/null +++ b/experimental/dbx2dab/.python-version @@ -0,0 +1 @@ +3.11 diff --git a/experimental/dbx2dab/README.md b/experimental/dbx2dab/README.md new file mode 100644 index 00000000..abcd57aa --- /dev/null +++ b/experimental/dbx2dab/README.md @@ -0,0 +1,10 @@ +# dbx2dabs + +Requires [uv](https://docs.astral.sh/uv/). + +Usage: +``` +uv run ./main.py DIR_TO_DBX_DIRECTORY +``` + +This writes DAB configuration to the same directory. diff --git a/experimental/dbx2dab/dbx2dab/__init__.py b/experimental/dbx2dab/dbx2dab/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/experimental/dbx2dab/dbx2dab/compare.py b/experimental/dbx2dab/dbx2dab/compare.py new file mode 100644 index 00000000..b6cd7da6 --- /dev/null +++ b/experimental/dbx2dab/dbx2dab/compare.py @@ -0,0 +1,393 @@ +from typing import Dict, List + + +def recursive_intersection( + dict1: Dict[str, any], dict2: Dict[str, any] +) -> Dict[str, any]: + """ + Compute the recursive intersection of two dictionaries. + + Args: + dict1 (dict): First dictionary. + dict2 (dict): Second dictionary. + + Returns: + dict: A new dictionary containing the intersection. + """ + intersection = {} + for key in dict1: + if key in dict2: + value1 = dict1[key] + value2 = dict2[key] + + if isinstance(value1, dict) and isinstance(value2, dict): + nested = recursive_intersection(value1, value2) + if nested: + intersection[key] = nested + elif isinstance(value1, list) and isinstance(value2, list): + common = intersect_lists(value1, value2) + if common: + intersection[key] = common + else: + if value1 == value2: + intersection[key] = value1 + return intersection + + +def lists_have_key(list1: List[any], list2: List[any], key: str) -> bool: + """ + Check if two lists contain job clusters. + + Args: + list1 (list): First list. + list2 (list): Second list. + + Returns: + bool: True if both lists contain job clusters, False otherwise. + """ + keys1 = [item.get(key) if isinstance(item, dict) else None for item in list1] + keys2 = [item.get(key) if isinstance(item, dict) else None for item in list2] + return all(keys1) and all(keys2) + + +def intersect_lists_with_key(list1: List[any], list2: List[any], key: str): + result = [] + for item1 in list1: + for item2 in list2: + if item1.get(key) == item2.get(key): + nested = recursive_intersection(item1, item2) + if nested: + result.append(nested) + return result + + +def subtract_lists_with_key(list1: List[any], list2: List[any], key: str): + result = [] + for item1 in list1: + found = False + for item2 in list2: + if item1.get(key) == item2.get(key): + found = True + nested = recursive_subtract_dict(item1, item2) + if nested: + out = {key: item1[key]} + out.update(nested) + result.append(out) + + if not found: + result.append(item1) + return result + + +def are_job_cluster_lists(list1: List[any], list2: List[any]) -> bool: + return lists_have_key(list1, list2, "job_cluster_key") + + +def are_task_lists(list1: List[any], list2: List[any]) -> bool: + return lists_have_key(list1, list2, "task_key") + + +def intersect_lists(list1: List[any], list2: List[any]): + """ + Compute the intersection of two lists, handling dictionaries within lists. + + Args: + list1 (list): First list. + list2 (list): Second list. + + Returns: + list: A list containing the intersecting elements. + """ + result = [] + + if lists_have_key(list1, list2, "task_key"): + return intersect_lists_with_key(list1, list2, "task_key") + + if lists_have_key(list1, list2, "job_cluster_key"): + return intersect_lists_with_key(list1, list2, "job_cluster_key") + + # Generic intersection + for item1, item2 in zip(list1, list2): + if item1 is None or item2 is None: + break + + if isinstance(item1, dict) and isinstance(item2, dict): + if recursive_compare(item1, item2): + result.append(item1) + else: + if item1 == item2: + result.append(item1) + + return result + + +def recursive_compare(d1: Dict[str, any], d2: Dict[str, any]): + """ + Recursively compare two dictionaries for equality. + + Args: + d1 (dict): First dictionary. + d2 (dict): Second dictionary. + + Returns: + bool: True if dictionaries are equal, False otherwise. + """ + if d1.keys() != d2.keys(): + return False + for key in d1: + v1 = d1[key] + v2 = d2[key] + if isinstance(v1, dict) and isinstance(v2, dict): + if not recursive_compare(v1, v2): + return False + elif isinstance(v1, list) and isinstance(v2, list): + if not intersect_lists(v1, v2) == v1: + return False + else: + if v1 != v2: + return False + return True + + +def recursive_subtract(o1: any, o2: any) -> any: + """ + Compute the recursive subtraction of two objects. + + Args: + o1 (any): First object. + o2 (any): Second object. + + Returns: + any: The subtracted object. + """ + if isinstance(o1, dict) and isinstance(o2, dict): + return recursive_subtract_dict(o1, o2) + elif isinstance(o1, list) and isinstance(o2, list): + return recursive_subtract_list(o1, o2) + else: + raise ValueError("Unsupported types for subtraction") + + +def recursive_subtract_dict( + dict1: Dict[str, any], dict2: Dict[str, any] +) -> Dict[str, any]: + """ + Compute the recursive subtraction of two dictionaries. + + Args: + dict1 (dict): First dictionary. + dict2 (dict): Second dictionary. + + Returns: + dict: A new dictionary containing the subtraction. + """ + subtraction = {} + for key in dict1: + if key not in dict2: + subtraction[key] = dict1[key] + else: + value1 = dict1[key] + value2 = dict2[key] + + if isinstance(value1, dict) and isinstance(value2, dict): + nested = recursive_subtract(value1, value2) + if nested: + subtraction[key] = nested + elif isinstance(value1, list) and isinstance(value2, list): + common = recursive_subtract_list(value1, value2) + if common: + subtraction[key] = common + else: + if value1 != value2: + subtraction[key] = value1 + return subtraction + + +def recursive_subtract_list(list1: List[any], list2: List[any]): + """ + Compute the subtraction of two lists, handling dictionaries within lists. + + Args: + list1 (list): First list. + list2 (list): Second list. + + Returns: + list: A list containing the subtracted elements. + """ + result = [] + + if lists_have_key(list1, list2, "task_key"): + return subtract_lists_with_key(list1, list2, "task_key") + + if lists_have_key(list1, list2, "job_cluster_key"): + return subtract_lists_with_key(list1, list2, "job_cluster_key") + + # If both lists contain job clusters, compute the subtraction + # where the job cluster keys are equal. + if are_job_cluster_lists(list1, list2): + for item1 in list1: + found = False + for item2 in list2: + if item1.get("job_cluster_key") == item2.get("job_cluster_key"): + found = True + nested = recursive_subtract_dict(item1, item2) + if nested: + result.append(nested) + + if not found: + result.append(item1) + return result + + for item1, item2 in zip(list1, list2): + if item1 is None or item2 is None: + break + + if isinstance(item1, dict) and isinstance(item2, dict): + if not recursive_compare(item1, item2): + result.append(item1) + else: + if item1 != item2: + result.append(item1) + + return result + + +class Walker: + _insert_callback = None + _update_callback = None + _delete_callback = None + + def __init__( + self, insert_callback=None, update_callback=None, delete_callback=None + ): + self._insert_callback = insert_callback + self._update_callback = update_callback + self._delete_callback = delete_callback + + def insert_callback(self, path, key, value): + if self._insert_callback: + self._insert_callback(path, key, value) + return + raise ValueError(f"Insert: {path}: {key}={value}") + + def update_callback(self, path, old_value, new_value): + if self._update_callback: + self._update_callback(path, old_value, new_value) + return + raise ValueError(f"Update: {path}: {old_value} -> {new_value}") + + def delete_callback(self, path, key, value): + if self._delete_callback: + self._delete_callback(path, key, value) + return + raise ValueError(f"Delete: {path}: {key}={value}") + + def walk(self, o1, o2, path=None): + if path is None: + path = [] + + if isinstance(o1, dict) and isinstance(o2, dict): + return self._walk_dict(o1, o2, path) + elif isinstance(o1, list) and isinstance(o2, list): + return self._walk_list(o1, o2, path) + else: + return self._walk_scalar(o1, o2, path) + + def _walk_dict(self, o1, o2, path): + for key in o1: + if key not in o2: + self.delete_callback(path, key, o1[key]) + else: + self.walk(o1[key], o2[key], path + [key]) + + for key in o2: + if key not in o1: + self.insert_callback(path, key, o2[key]) + + def _walk_list(self, o1, o2, path): + for i, item in enumerate(o1): + if i >= len(o2): + self.delete_callback(path, i, item) + else: + self.walk(item, o2[i], path + [i]) + + for i in range(len(o1), len(o2)): + self.insert_callback(path, i, o2[i]) + + def _walk_scalar(self, o1, o2, path): + if o1 != o2: + self.update_callback(path, o1, o2) + + +def walk(o1, o2, insert_callback=None, update_callback=None, delete_callback=None): + walker = Walker(insert_callback, update_callback, delete_callback) + walker.walk(o1, o2) + + +def recursive_merge(o1: any, o2: any) -> any: + """ + Compute the recursive merge of two objects. + + Args: + o1 (any): First object. + o2 (any): Second object. + + Returns: + any: The merged object. + """ + if isinstance(o1, dict) and isinstance(o2, dict): + return recursive_merge_dict(o1, o2) + elif isinstance(o1, list) and isinstance(o2, list): + return recursive_merge_list(o1, o2) + else: + raise ValueError("Unsupported types for merge") + + +def recursive_merge_dict( + dict1: Dict[str, any], dict2: Dict[str, any] +) -> Dict[str, any]: + """ + Compute the recursive merge of two dictionaries. + + Args: + dict1 (dict): First dictionary. + dict2 (dict): Second dictionary. + + Returns: + dict: A new dictionary containing the merge. + """ + merged = dict(dict1) + for key in dict2: + if key in merged: + value1 = dict1[key] + value2 = dict2[key] + + if isinstance(value1, dict) and isinstance(value2, dict): + merged[key] = recursive_merge(value1, value2) + elif isinstance(value1, list) and isinstance(value2, list): + merged[key] = recursive_merge_list(value1, value2) + else: + merged[key] = value2 + else: + merged[key] = dict2[key] + return merged + + +def recursive_merge_list(list1: List[any], list2: List[any]): + """ + Compute the merge of two lists, handling dictionaries within lists. + + Args: + list1 (list): First list. + list2 (list): Second list. + + Returns: + list: A list containing the merged elements. + """ + merged = list(list1) + for i, item in enumerate(list2): + if i < len(merged): + merged[i] = recursive_merge(merged[i], item) + else: + merged.append(item) + return merged diff --git a/experimental/dbx2dab/dbx2dab/config_reader.py b/experimental/dbx2dab/dbx2dab/config_reader.py new file mode 100644 index 00000000..bd60e64f --- /dev/null +++ b/experimental/dbx2dab/dbx2dab/config_reader.py @@ -0,0 +1,136 @@ +import json +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Any, Dict, Optional + +import jinja2 +import yaml + + +class _AbstractConfigReader(ABC): + def __init__(self, path: Path): + self._path = path + self.config = self.get_config() + + def get_config(self) -> any: + return self._read_file() + + @abstractmethod + def _read_file(self) -> any: + """""" + + +class YamlConfigReader(_AbstractConfigReader): + def _read_file(self) -> any: + return yaml.load(self._path.read_text(encoding="utf-8"), yaml.SafeLoader) + + +class JsonConfigReader(_AbstractConfigReader): + def _read_file(self) -> any: + return json.loads(self._path.read_text(encoding="utf-8")) + + +class Jinja2ConfigReader(_AbstractConfigReader): + def __init__( + self, + path: Path, + ext: str, + jinja_vars_file: Optional[Path], + env_proxy=None, + var_proxy=None, + ): + self._ext = ext + self._jinja_vars_file = jinja_vars_file + self._env_proxy = env_proxy + self._var_proxy = var_proxy + super().__init__(path) + + @staticmethod + def _read_vars_file(file_path: Path) -> Dict[str, Any]: + return yaml.load(file_path.read_text(encoding="utf-8"), yaml.SafeLoader) + + @classmethod + def _render_content(cls, file_path: Path, globals: Dict[str, Any]) -> str: + absolute_parent_path = file_path.absolute().parent + file_name = file_path.name + + env = jinja2.Environment(loader=jinja2.FileSystemLoader(absolute_parent_path)) + template = env.get_template(file_name) + + return template.render(**globals) + + def _read_file(self) -> any: + rendered = self._render_content( + self._path, + { + "env": self._env_proxy, + "var": self._var_proxy, + }, + ) + + if self._ext == ".json": + _content = json.loads(rendered) + return _content + elif self._ext in [".yml", ".yaml"]: + _content = yaml.load(rendered, yaml.SafeLoader) + return _content + else: + raise Exception(f"Unexpected extension for Jinja reader: {self._ext}") + + +class ConfigReader: + """ + Entrypoint for reading the raw configurations from files. + In most cases there is no need to use the lower-level config readers. + If a new reader is introduced, it shall be used via the :code:`_define_reader` method. + """ + + def __init__( + self, + path: Path, + jinja_vars_file: Optional[Path] = None, + env_proxy=None, + var_proxy=None, + ): + self._jinja_vars_file = jinja_vars_file + self._path = path + self._env_proxy = env_proxy + self._var_proxy = var_proxy + self._reader = self._define_reader() + + def _define_reader(self) -> _AbstractConfigReader: + if False: + pass + # if len(self._path.suffixes) > 1: + # if self._path.suffixes[0] in [".json", ".yaml", ".yml"] and self._path.suffixes[1] == ".j2": + # dbx_echo( + # """[bright_magenta bold]You're using a deployment file with .j2 extension. + # if you would like to use Jinja directly inside YAML or JSON files without changing the extension, + # you can also configure your project to support in-place Jinja by running: + # [code]dbx configure --enable-inplace-jinja-support[/code][/bright_magenta bold]""" + # ) + # return Jinja2ConfigReader(self._path, ext=self._path.suffixes[0], jinja_vars_file=self._jinja_vars_file) + # elif ProjectConfigurationManager().get_jinja_support(): + # return Jinja2ConfigReader(self._path, ext=self._path.suffixes[0], jinja_vars_file=self._jinja_vars_file) + else: + if self._jinja_vars_file: + return Jinja2ConfigReader( + self._path, + ext=self._path.suffixes[0], + jinja_vars_file=self._jinja_vars_file, + env_proxy=self._env_proxy, + var_proxy=self._var_proxy, + ) + if self._path.suffixes[0] == ".json": + return JsonConfigReader(self._path) + elif self._path.suffixes[0] in [".yaml", ".yml"]: + return YamlConfigReader(self._path) + + # no matching reader found, raising an exception + raise Exception( + f"Unexpected extension of the deployment file: {self._path}. " + f"Please check the documentation for supported extensions." + ) + + def get_config(self) -> any: + return self._reader.config diff --git a/experimental/dbx2dab/dbx2dab/loader.py b/experimental/dbx2dab/dbx2dab/loader.py new file mode 100644 index 00000000..4ce1db66 --- /dev/null +++ b/experimental/dbx2dab/dbx2dab/loader.py @@ -0,0 +1,177 @@ +from pathlib import Path +from typing import Dict, List, Tuple +import yaml + +from dbx2dab.config_reader import ConfigReader +from dbx2dab.compare import walk + + +class VerboseSafeDumper(yaml.SafeDumper): + def ignore_aliases(self, data): + return True + + +class EnvProxy: + def __init__(self): + self.items = [] + pass + + def variable_name(self, name: str): + return f"ENV_{name}".upper() + + def variables(self): + return [self.variable_name(item) for item in self.items] + + def interpolation_for_item(self, name: str): + return f"${{var.{self.variable_name(name)}}}" + + def __getitem__(self, item): + self.items.append(item) + return self.interpolation_for_item(item) + + +class VarProxy: + def __init__(self, passthrough: Dict[str, any]): + self.items = [] + self.passthrough = passthrough + pass + + def variable_name(self, name: str): + return name.upper() + + def variables(self): + return [self.variable_name(item) for item in self.items] + + def interpolation_for_item(self, name: str): + return f"${{var.{self.variable_name(name)}}}" + + def __getitem__(self, item): + if item in self.passthrough: + return self.passthrough[item] + + self.items.append(item) + return self.interpolation_for_item(item) + + +class Loader: + def __init__(self, path: Path): + self._path = path + + def detect_environments(self) -> List[str]: + """ + Expect the following directory structure: + - conf/ + - dev/ + - config.yaml + - stg/ + - config.yaml + - prod/ + - config.yaml + + This function will return ["dev", "stg", "prod"] + """ + return [ + item.name for item in self._path.joinpath("conf").iterdir() if item.is_dir() + ] + + def _get_config(self, environment: str, env_proxy, var_proxy) -> any: + deployment_file = self._path.joinpath("conf/deployment.yaml") + variables_file = self._path.joinpath(f"conf/{environment}/config.yaml") + + config_reader = ConfigReader( + deployment_file, + variables_file, + env_proxy=env_proxy, + var_proxy=var_proxy, + ) + + config = config_reader.get_config() + return config["environments"][environment] + + def load_variables_for_environment(self, environment: str) -> Dict[str, any]: + file = self._path.joinpath(f"conf/{environment}/config.yaml") + obj: Dict[str, any] = yaml.load( + file.read_text(encoding="utf-8"), yaml.SafeLoader + ) + return obj + + def compute_variable_allowlist(self, environment: str) -> List[str]: + variables_ref = self.load_variables_for_environment(environment) + env_proxy = EnvProxy() + + # Interpolate all variables. + complete_config = self._get_config( + environment, EnvProxy(), VarProxy(passthrough=variables_ref) + ) + + # We will try to find the impact of each variable on the configuration. + # All variables that can be replaced by a bundle variable will be added to the allowlist. + variable_allowlist = [] + + # Now look for the impact of each variable + for key in variables_ref.keys(): + dup = dict(variables_ref) + dup.pop(key) + + # If we cannot use a bundle variable for this key, continue. + try: + var_proxy = VarProxy(passthrough=dup) + config = self._get_config(environment, env_proxy, var_proxy) + except Exception: + continue + + # If we can use a bundle variable for this key, look at the difference. + # We care that every use of the variable is replaced by the bundle variable. + # If it is replaced by something else, there is logic we cannot capture. + correctly_replaced = True + + def update_callback(path, old_value, new_value): + nonlocal correctly_replaced + if ( + isinstance(new_value, str) + and var_proxy.interpolation_for_item(key) in new_value + ): + return + print( + f"Detected incompatible replacement for {key}: {path}: {old_value} -> {new_value}" + ) + correctly_replaced = False + return + + walk(complete_config, config, update_callback=update_callback) + + # If we found an incompatible replacement, bail. + if not correctly_replaced: + continue + + variable_allowlist.append(key) + + return variable_allowlist + + def load_for_environment(self, environment: str) -> Tuple[any, EnvProxy, VarProxy]: + variables_ref = self.load_variables_for_environment(environment) + variable_allowlist = self.compute_variable_allowlist(environment) + + # Remove all variables that are not in the allowlist. + for key in variable_allowlist: + variables_ref.pop(key) + + env_proxy = EnvProxy() + var_proxy = VarProxy(passthrough=variables_ref) + config = self._get_config(environment, env_proxy, var_proxy) + return config, env_proxy, var_proxy + + +if __name__ == "__main__": + """ + Below is an example of how to use the Loader class. + + Use this for testing and debugging purposes only. + """ + + loader = Loader( + Path(__file__).parent / "../../databricks-etl-notebook-template-main" + ) + + for env in ["dev", "stg", "prod"]: + loader.load_for_environment(env) diff --git a/experimental/dbx2dab/main.py b/experimental/dbx2dab/main.py new file mode 100644 index 00000000..4516e7b7 --- /dev/null +++ b/experimental/dbx2dab/main.py @@ -0,0 +1,253 @@ +import argparse +import sys +import re +from pathlib import Path +from typing import Dict + +import yaml +import jinja2 + +from dbx2dab.compare import ( + recursive_compare, + recursive_intersection, + recursive_subtract, + recursive_merge, +) + +from dbx2dab.loader import Loader + + +class VerboseSafeDumper(yaml.SafeDumper): + """ + A YAML dumper that does not use aliases. + """ + + def ignore_aliases(self, data): + return True + + +class Job: + def __init__(self, name: str) -> None: + self.name = name + self.configs = dict() + + def normalized_key(self) -> str: + name = str(self.name) + # Remove ${foo.bar} with a regex + name = re.sub(r"\${.*?}", "", name) + # Remove leading and trailing underscores + name = re.sub(r"^_+|_+$", "", name) + return name + + def register_configuration(self, environment: str, config: Dict[str, any]) -> None: + self.configs[environment] = config + + def all_equal(self) -> bool: + keys = list(self.configs.keys()) + if len(keys) == 1: + return True + + for i in range(1, len(keys)): + if not recursive_compare(self.configs[keys[i - 1]], self.configs[keys[i]]): + return False + + return True + + def compute_base(self) -> Dict[str, any]: + keys = list(self.configs.keys()) + if len(keys) == 1: + return self.configs[keys[0]] + + out = self.configs[keys[0]] + for key in keys[1:]: + out = recursive_intersection(out, self.configs[key]) + return out + + def compute_resource_definition(self) -> Dict[str, any]: + ordered_keys = [ + "name", + "tags", + "schedule", + "email_notifications", + "git_source", + "tasks", + "job_clusters", + ] + + obj = self.compute_base() + + return { + "resources": { + "jobs": {self.normalized_key(): {k: obj[k] for k in ordered_keys}} + } + } + + def compute_override_for_environment(self, environment: str) -> Dict[str, any]: + base = self.compute_base() + if environment not in self.configs: + return {} + + config = self.configs[environment] + override = recursive_subtract(config, base) + + # If the configuration is the same as the base, we don't need to override. + if not override: + return {} + + return { + "targets": { + environment: { + "resources": { + "jobs": { + self.normalized_key(): recursive_subtract(config, base) + } + } + } + } + } + + +def dedup_variables(variables): + deduped = dict() + for v in variables: + if v not in deduped: + deduped[v] = None + return deduped.keys() + + +def save_databricks_yml(base_path: Path, env_variables, var_variables): + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(Path(__file__).parent.joinpath("templates")) + ) + template = env.get_template("databricks.yml.j2") + + base_name = base_path.name + dst = base_path.joinpath("databricks.yml") + print("Writing: ", dst) + with open(dst, "w") as f: + f.write( + template.render( + bundle_name=base_name, + env_variables=env_variables, + var_variables=var_variables, + ) + ) + + +def compute_variables_for_environment( + environment: str, variables: Dict[str, any] +) -> Dict[str, any]: + return { + "targets": { + environment: { + "variables": variables, + } + } + } + + +def main(): + parser = argparse.ArgumentParser(description="Generate Databricks configurations") + parser.add_argument("dir", help="Path to the DBX project") + parser.add_argument( + "-v", "--verbose", action="store_true", help="Print verbose output" + ) + args = parser.parse_args() + verbose: bool = args.verbose + + base_path = Path(args.dir) + loader = Loader(base_path) + envs = loader.detect_environments() + print("Detected environments:", envs) + + env_variables = [] + var_variables = [] + + jobs: Dict[str, Job] = dict() + for env in envs: + config, env_proxy, var_proxy = loader.load_for_environment(env) + env_variables.extend(env_proxy.variables()) + var_variables.extend(var_proxy.variables()) + for workflow in config["workflows"]: + name = workflow["name"] + if name not in jobs: + jobs[name] = Job(name) + + jobs[name].register_configuration(env, workflow) + + for job in jobs.values(): + base_job = job.compute_base() + + if verbose: + print("Job:", job.name) + + # Write job configuration to "./resources" directory + resource_path = base_path.joinpath("resources") + resource_path.mkdir(exist_ok=True) + + dst = resource_path.joinpath(f"{job.normalized_key()}.yml") + print("Writing: ", dst) + with open(dst, "w") as f: + yaml.dump( + job.compute_resource_definition(), + f, + Dumper=VerboseSafeDumper, + sort_keys=False, + ) + + for environment, config in job.configs.items(): + diff = recursive_subtract(config, base_job) + + if verbose: + yaml.dump( + diff, + sys.stdout, + indent=2, + Dumper=VerboseSafeDumper, + sort_keys=False, + ) + + # Write variable definitions + env_variables = dedup_variables(env_variables) + var_variables = dedup_variables(var_variables) + save_databricks_yml(base_path, env_variables, var_variables) + + # Write resource overrides + for env in envs: + out = {} + for job in jobs.values(): + out = recursive_merge(out, job.compute_override_for_environment(env)) + + if out: + dst = base_path.joinpath(f"conf/{env}/overrides.yml") + print("Writing: ", dst) + with open(dst, "w") as f: + yaml.dump( + out, + f, + Dumper=VerboseSafeDumper, + sort_keys=False, + ) + + # Write variable values + for env in envs: + variables = loader.load_variables_for_environment(env) + + # Include only variables listed in "var_variables" + variables = {k: v for k, v in variables.items() if k in var_variables} + + out = compute_variables_for_environment(env, variables) + if out: + dst = base_path.joinpath(f"conf/{env}/variables.yml") + print("Writing: ", dst) + with open(dst, "w") as f: + yaml.dump( + out, + f, + Dumper=VerboseSafeDumper, + sort_keys=False, + ) + + +if __name__ == "__main__": + main() diff --git a/experimental/dbx2dab/pyproject.toml b/experimental/dbx2dab/pyproject.toml new file mode 100644 index 00000000..319f2373 --- /dev/null +++ b/experimental/dbx2dab/pyproject.toml @@ -0,0 +1,11 @@ +[project] +name = "eval" +version = "0.1.0" +description = "Add your description here" +requires-python = ">=3.11" +dependencies = [ + "pyyaml", + "jinja2", + "pytest", + "ruff", +] diff --git a/experimental/dbx2dab/templates/databricks.yml.j2 b/experimental/dbx2dab/templates/databricks.yml.j2 new file mode 100644 index 00000000..9d9ae08f --- /dev/null +++ b/experimental/dbx2dab/templates/databricks.yml.j2 @@ -0,0 +1,31 @@ +bundle: + name: {{ bundle_name }} + +include: + - "resources/*.yml" + - "conf/*/overrides.yml" + - "conf/*/variables.yml" + +variables: + # Variables that are specified as environment variables. + # These are known only at deployment time. + # + # These were directly accessible in DBX configuration. + # This is not possible in bundles for security reasons. + # + # A variable that was previously referred as env["CI_PROJECT_URL"], + # must now be used as ${var.env_ci_project_url} and can be set + # with the environment variable "BUNDLE_VAR_env_ci_project_url". + # + {% for item in env_variables -%} + {{ item }}: + description: "" + {% endfor %} + + # Variables with known values at deployment time. + # They may define a default below, may be set for each target, + # or may be set at deployment time. + {% for item in var_variables -%} + {{ item }}: + description: "" + {% endfor %} diff --git a/experimental/dbx2dab/tests/test_compare.py b/experimental/dbx2dab/tests/test_compare.py new file mode 100644 index 00000000..920509d7 --- /dev/null +++ b/experimental/dbx2dab/tests/test_compare.py @@ -0,0 +1,190 @@ +from dbx2dab.compare import recursive_subtract + + +def test_recursive_subtract_equal(): + a = { + "a": 1, + "b": 2, + "c": { + "d": 3, + "e": 4, + "f": { + "g": 5, + }, + }, + } + b = { + "a": 1, + "b": 2, + "c": { + "d": 3, + "e": 4, + "f": { + "g": 5, + }, + }, + } + assert recursive_subtract(a, b) == {} + + +def test_recursive_subtract_nested_element(): + a = { + "a": 1, + "b": 2, + "c": { + "d": 3, + "e": 4, + "f": { + "g": 5, + }, + }, + } + b = { + "a": 1, + "b": 2, + "c": { + "d": 3, + "e": 4, + "f": { + "g": 6, + }, + }, + } + assert recursive_subtract(a, b) == {"c": {"f": {"g": 5}}} + + +def test_recursive_subtract_superset(): + a = { + "a": 1, + "b": 2, + "c": { + "d": 3, + "e": 4, + "f": { + "g": 5, + }, + }, + } + b = { + "a": 1, + "b": 2, + "c": { + "d": 3, + "e": 4, + "f": { + "g": 5, + "h": 6, + }, + }, + } + assert recursive_subtract(a, b) == {} + + +def test_recursive_subtract_list_job_clusters(): + a = [ + { + "job_cluster_key": "cluster1", + "node_type_id": "Standard_DS3_v2", + "spark_version": "14.3.x-scala2.12", + "num_workers": 1, + }, + { + "job_cluster_key": "cluster2", + "node_type_id": "Standard_DS3_v2", + "spark_version": "14.3.x-scala2.12", + "num_workers": 1, + }, + ] + + b = [ + { + "job_cluster_key": "cluster1", + "node_type_id": "Standard_DS3_v2", + "spark_version": "14.3.x-scala2.12", + "num_workers": 1, + }, + { + "job_cluster_key": "cluster2", + "node_type_id": "Standard_DS3_v2", + "spark_version": "14.3.x-scala2.12", + "num_workers": 1, + }, + ] + + assert recursive_subtract(a, b) == [] + +def test_recursive_subtract_list_job_clusters_reverse_order(): + a = [ + { + "job_cluster_key": "cluster1", + "node_type_id": "Standard_DS3_v2", + "spark_version": "14.3.x-scala2.12", + "num_workers": 1, + }, + { + "job_cluster_key": "cluster2", + "node_type_id": "Standard_DS3_v2", + "spark_version": "14.3.x-scala2.12", + "num_workers": 1, + }, + ] + + b = [ + { + "job_cluster_key": "cluster2", + "node_type_id": "Standard_DS3_v2", + "spark_version": "14.3.x-scala2.12", + "num_workers": 1, + }, + { + "job_cluster_key": "cluster1", + "node_type_id": "Standard_DS3_v2", + "spark_version": "14.3.x-scala2.12", + "num_workers": 1, + }, + ] + + assert recursive_subtract(a, b) == [] + + +def test_recursive_subtract_list_job_clusters_nominal(): + a = [ + { + "job_cluster_key": "cluster1", + "node_type_id": "Standard_DS3_v2", + "spark_version": "14.3.x-scala2.12", + "num_workers": 2, + }, + { + "job_cluster_key": "cluster2", + "node_type_id": "Standard_DS3_v2", + "spark_version": "14.3.x-scala2.12", + "num_workers": 2, + } + ] + + b = [ + { + "job_cluster_key": "cluster1", + "node_type_id": "Standard_DS3_v2", + "spark_version": "14.3.x-scala2.12", + "num_workers": 1, + }, + { + "job_cluster_key": "cluster2", + "node_type_id": "Standard_DS3_v2", + "spark_version": "14.3.x-scala2.12", + "num_workers": 1, + }, + ] + + assert recursive_subtract(a, b) == [ + { + "job_cluster_key": "cluster1", + "num_workers": 2, + }, + { + "job_cluster_key": "cluster2", + "num_workers": 2, + } + ] diff --git a/experimental/dbx2dab/uv.lock b/experimental/dbx2dab/uv.lock new file mode 100644 index 00000000..9cff7cde --- /dev/null +++ b/experimental/dbx2dab/uv.lock @@ -0,0 +1,192 @@ +version = 1 +requires-python = ">=3.11" + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335 }, +] + +[[package]] +name = "eval" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "jinja2" }, + { name = "pytest" }, + { name = "pyyaml" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "jinja2" }, + { name = "pytest" }, + { name = "pyyaml" }, + { name = "ruff" }, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/4b/cbd8e699e64a6f16ca3a8220661b5f83792b3017d0f79807cb8708d33913/iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3", size = 4646 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374", size = 5892 }, +] + +[[package]] +name = "jinja2" +version = "3.1.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ed/55/39036716d19cab0747a5020fc7e907f362fbf48c984b14e62127f7e68e5d/jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369", size = 240245 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/31/80/3a54838c3fb461f6fec263ebf3a3a41771bd05190238de3486aae8540c36/jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d", size = 133271 }, +] + +[[package]] +name = "markupsafe" +version = "3.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/97/5d42485e71dfc078108a86d6de8fa46db44a1a9295e89c5d6d4a06e23a62/markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0", size = 20537 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6b/28/bbf83e3f76936960b850435576dd5e67034e200469571be53f69174a2dfd/MarkupSafe-3.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9025b4018f3a1314059769c7bf15441064b2207cb3f065e6ea1e7359cb46db9d", size = 14353 }, + { url = "https://files.pythonhosted.org/packages/6c/30/316d194b093cde57d448a4c3209f22e3046c5bb2fb0820b118292b334be7/MarkupSafe-3.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:93335ca3812df2f366e80509ae119189886b0f3c2b81325d39efdb84a1e2ae93", size = 12392 }, + { url = "https://files.pythonhosted.org/packages/f2/96/9cdafba8445d3a53cae530aaf83c38ec64c4d5427d975c974084af5bc5d2/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cb8438c3cbb25e220c2ab33bb226559e7afb3baec11c4f218ffa7308603c832", size = 23984 }, + { url = "https://files.pythonhosted.org/packages/f1/a4/aefb044a2cd8d7334c8a47d3fb2c9f328ac48cb349468cc31c20b539305f/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a123e330ef0853c6e822384873bef7507557d8e4a082961e1defa947aa59ba84", size = 23120 }, + { url = "https://files.pythonhosted.org/packages/8d/21/5e4851379f88f3fad1de30361db501300d4f07bcad047d3cb0449fc51f8c/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e084f686b92e5b83186b07e8a17fc09e38fff551f3602b249881fec658d3eca", size = 23032 }, + { url = "https://files.pythonhosted.org/packages/00/7b/e92c64e079b2d0d7ddf69899c98842f3f9a60a1ae72657c89ce2655c999d/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8213e09c917a951de9d09ecee036d5c7d36cb6cb7dbaece4c71a60d79fb9798", size = 24057 }, + { url = "https://files.pythonhosted.org/packages/f9/ac/46f960ca323037caa0a10662ef97d0a4728e890334fc156b9f9e52bcc4ca/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5b02fb34468b6aaa40dfc198d813a641e3a63b98c2b05a16b9f80b7ec314185e", size = 23359 }, + { url = "https://files.pythonhosted.org/packages/69/84/83439e16197337b8b14b6a5b9c2105fff81d42c2a7c5b58ac7b62ee2c3b1/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0bff5e0ae4ef2e1ae4fdf2dfd5b76c75e5c2fa4132d05fc1b0dabcd20c7e28c4", size = 23306 }, + { url = "https://files.pythonhosted.org/packages/9a/34/a15aa69f01e2181ed8d2b685c0d2f6655d5cca2c4db0ddea775e631918cd/MarkupSafe-3.0.2-cp311-cp311-win32.whl", hash = "sha256:6c89876f41da747c8d3677a2b540fb32ef5715f97b66eeb0c6b66f5e3ef6f59d", size = 15094 }, + { url = "https://files.pythonhosted.org/packages/da/b8/3a3bd761922d416f3dc5d00bfbed11f66b1ab89a0c2b6e887240a30b0f6b/MarkupSafe-3.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:70a87b411535ccad5ef2f1df5136506a10775d267e197e4cf531ced10537bd6b", size = 15521 }, + { url = "https://files.pythonhosted.org/packages/22/09/d1f21434c97fc42f09d290cbb6350d44eb12f09cc62c9476effdb33a18aa/MarkupSafe-3.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf", size = 14274 }, + { url = "https://files.pythonhosted.org/packages/6b/b0/18f76bba336fa5aecf79d45dcd6c806c280ec44538b3c13671d49099fdd0/MarkupSafe-3.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225", size = 12348 }, + { url = "https://files.pythonhosted.org/packages/e0/25/dd5c0f6ac1311e9b40f4af06c78efde0f3b5cbf02502f8ef9501294c425b/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c99d261bd2d5f6b59325c92c73df481e05e57f19837bdca8413b9eac4bd8028", size = 24149 }, + { url = "https://files.pythonhosted.org/packages/f3/f0/89e7aadfb3749d0f52234a0c8c7867877876e0a20b60e2188e9850794c17/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e17c96c14e19278594aa4841ec148115f9c7615a47382ecb6b82bd8fea3ab0c8", size = 23118 }, + { url = "https://files.pythonhosted.org/packages/d5/da/f2eeb64c723f5e3777bc081da884b414671982008c47dcc1873d81f625b6/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88416bd1e65dcea10bc7569faacb2c20ce071dd1f87539ca2ab364bf6231393c", size = 22993 }, + { url = "https://files.pythonhosted.org/packages/da/0e/1f32af846df486dce7c227fe0f2398dc7e2e51d4a370508281f3c1c5cddc/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2181e67807fc2fa785d0592dc2d6206c019b9502410671cc905d132a92866557", size = 24178 }, + { url = "https://files.pythonhosted.org/packages/c4/f6/bb3ca0532de8086cbff5f06d137064c8410d10779c4c127e0e47d17c0b71/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:52305740fe773d09cffb16f8ed0427942901f00adedac82ec8b67752f58a1b22", size = 23319 }, + { url = "https://files.pythonhosted.org/packages/a2/82/8be4c96ffee03c5b4a034e60a31294daf481e12c7c43ab8e34a1453ee48b/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ad10d3ded218f1039f11a75f8091880239651b52e9bb592ca27de44eed242a48", size = 23352 }, + { url = "https://files.pythonhosted.org/packages/51/ae/97827349d3fcffee7e184bdf7f41cd6b88d9919c80f0263ba7acd1bbcb18/MarkupSafe-3.0.2-cp312-cp312-win32.whl", hash = "sha256:0f4ca02bea9a23221c0182836703cbf8930c5e9454bacce27e767509fa286a30", size = 15097 }, + { url = "https://files.pythonhosted.org/packages/c1/80/a61f99dc3a936413c3ee4e1eecac96c0da5ed07ad56fd975f1a9da5bc630/MarkupSafe-3.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:8e06879fc22a25ca47312fbe7c8264eb0b662f6db27cb2d3bbbc74b1df4b9b87", size = 15601 }, + { url = "https://files.pythonhosted.org/packages/83/0e/67eb10a7ecc77a0c2bbe2b0235765b98d164d81600746914bebada795e97/MarkupSafe-3.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ba9527cdd4c926ed0760bc301f6728ef34d841f405abf9d4f959c478421e4efd", size = 14274 }, + { url = "https://files.pythonhosted.org/packages/2b/6d/9409f3684d3335375d04e5f05744dfe7e9f120062c9857df4ab490a1031a/MarkupSafe-3.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f8b3d067f2e40fe93e1ccdd6b2e1d16c43140e76f02fb1319a05cf2b79d99430", size = 12352 }, + { url = "https://files.pythonhosted.org/packages/d2/f5/6eadfcd3885ea85fe2a7c128315cc1bb7241e1987443d78c8fe712d03091/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:569511d3b58c8791ab4c2e1285575265991e6d8f8700c7be0e88f86cb0672094", size = 24122 }, + { url = "https://files.pythonhosted.org/packages/0c/91/96cf928db8236f1bfab6ce15ad070dfdd02ed88261c2afafd4b43575e9e9/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15ab75ef81add55874e7ab7055e9c397312385bd9ced94920f2802310c930396", size = 23085 }, + { url = "https://files.pythonhosted.org/packages/c2/cf/c9d56af24d56ea04daae7ac0940232d31d5a8354f2b457c6d856b2057d69/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3818cb119498c0678015754eba762e0d61e5b52d34c8b13d770f0719f7b1d79", size = 22978 }, + { url = "https://files.pythonhosted.org/packages/2a/9f/8619835cd6a711d6272d62abb78c033bda638fdc54c4e7f4272cf1c0962b/MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cdb82a876c47801bb54a690c5ae105a46b392ac6099881cdfb9f6e95e4014c6a", size = 24208 }, + { url = "https://files.pythonhosted.org/packages/f9/bf/176950a1792b2cd2102b8ffeb5133e1ed984547b75db47c25a67d3359f77/MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:cabc348d87e913db6ab4aa100f01b08f481097838bdddf7c7a84b7575b7309ca", size = 23357 }, + { url = "https://files.pythonhosted.org/packages/ce/4f/9a02c1d335caabe5c4efb90e1b6e8ee944aa245c1aaaab8e8a618987d816/MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:444dcda765c8a838eaae23112db52f1efaf750daddb2d9ca300bcae1039adc5c", size = 23344 }, + { url = "https://files.pythonhosted.org/packages/ee/55/c271b57db36f748f0e04a759ace9f8f759ccf22b4960c270c78a394f58be/MarkupSafe-3.0.2-cp313-cp313-win32.whl", hash = "sha256:bcf3e58998965654fdaff38e58584d8937aa3096ab5354d493c77d1fdd66d7a1", size = 15101 }, + { url = "https://files.pythonhosted.org/packages/29/88/07df22d2dd4df40aba9f3e402e6dc1b8ee86297dddbad4872bd5e7b0094f/MarkupSafe-3.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:e6a2a455bd412959b57a172ce6328d2dd1f01cb2135efda2e4576e8a23fa3b0f", size = 15603 }, + { url = "https://files.pythonhosted.org/packages/62/6a/8b89d24db2d32d433dffcd6a8779159da109842434f1dd2f6e71f32f738c/MarkupSafe-3.0.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:b5a6b3ada725cea8a5e634536b1b01c30bcdcd7f9c6fff4151548d5bf6b3a36c", size = 14510 }, + { url = "https://files.pythonhosted.org/packages/7a/06/a10f955f70a2e5a9bf78d11a161029d278eeacbd35ef806c3fd17b13060d/MarkupSafe-3.0.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a904af0a6162c73e3edcb969eeeb53a63ceeb5d8cf642fade7d39e7963a22ddb", size = 12486 }, + { url = "https://files.pythonhosted.org/packages/34/cf/65d4a571869a1a9078198ca28f39fba5fbb910f952f9dbc5220afff9f5e6/MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4aa4e5faecf353ed117801a068ebab7b7e09ffb6e1d5e412dc852e0da018126c", size = 25480 }, + { url = "https://files.pythonhosted.org/packages/0c/e3/90e9651924c430b885468b56b3d597cabf6d72be4b24a0acd1fa0e12af67/MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0ef13eaeee5b615fb07c9a7dadb38eac06a0608b41570d8ade51c56539e509d", size = 23914 }, + { url = "https://files.pythonhosted.org/packages/66/8c/6c7cf61f95d63bb866db39085150df1f2a5bd3335298f14a66b48e92659c/MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d16a81a06776313e817c951135cf7340a3e91e8c1ff2fac444cfd75fffa04afe", size = 23796 }, + { url = "https://files.pythonhosted.org/packages/bb/35/cbe9238ec3f47ac9a7c8b3df7a808e7cb50fe149dc7039f5f454b3fba218/MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6381026f158fdb7c72a168278597a5e3a5222e83ea18f543112b2662a9b699c5", size = 25473 }, + { url = "https://files.pythonhosted.org/packages/e6/32/7621a4382488aa283cc05e8984a9c219abad3bca087be9ec77e89939ded9/MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:3d79d162e7be8f996986c064d1c7c817f6df3a77fe3d6859f6f9e7be4b8c213a", size = 24114 }, + { url = "https://files.pythonhosted.org/packages/0d/80/0985960e4b89922cb5a0bac0ed39c5b96cbc1a536a99f30e8c220a996ed9/MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:131a3c7689c85f5ad20f9f6fb1b866f402c445b220c19fe4308c0b147ccd2ad9", size = 24098 }, + { url = "https://files.pythonhosted.org/packages/82/78/fedb03c7d5380df2427038ec8d973587e90561b2d90cd472ce9254cf348b/MarkupSafe-3.0.2-cp313-cp313t-win32.whl", hash = "sha256:ba8062ed2cf21c07a9e295d5b8a2a5ce678b913b45fdf68c32d95d6c1291e0b6", size = 15208 }, + { url = "https://files.pythonhosted.org/packages/4f/65/6079a46068dfceaeabb5dcad6d674f5f5c61a6fa5673746f42a9f4c233b3/MarkupSafe-3.0.2-cp313-cp313t-win_amd64.whl", hash = "sha256:e444a31f8db13eb18ada366ab3cf45fd4b31e4db1236a4448f68778c1d1a5a2f", size = 15739 }, +] + +[[package]] +name = "packaging" +version = "24.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/51/65/50db4dda066951078f0a96cf12f4b9ada6e4b811516bf0262c0f4f7064d4/packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002", size = 148788 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/aa/cc0199a5f0ad350994d660967a8efb233fe0416e4639146c089643407ce6/packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124", size = 53985 }, +] + +[[package]] +name = "pluggy" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/96/2d/02d4312c973c6050a18b314a5ad0b3210edb65a906f868e31c111dede4a6/pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1", size = 67955 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/5f/e351af9a41f866ac3f1fac4ca0613908d9a41741cfcf2228f4ad853b697d/pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669", size = 20556 }, +] + +[[package]] +name = "pytest" +version = "8.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/6c/62bbd536103af674e227c41a8f3dcd022d591f6eed5facb5a0f31ee33bbc/pytest-8.3.3.tar.gz", hash = "sha256:70b98107bd648308a7952b06e6ca9a50bc660be218d53c257cc1fc94fda10181", size = 1442487 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6b/77/7440a06a8ead44c7757a64362dd22df5760f9b12dc5f11b6188cd2fc27a0/pytest-8.3.3-py3-none-any.whl", hash = "sha256:a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2", size = 342341 }, +] + +[[package]] +name = "pyyaml" +version = "6.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", size = 130631 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/aa/7af4e81f7acba21a4c6be026da38fd2b872ca46226673c89a758ebdc4fd2/PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774", size = 184612 }, + { url = "https://files.pythonhosted.org/packages/8b/62/b9faa998fd185f65c1371643678e4d58254add437edb764a08c5a98fb986/PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee", size = 172040 }, + { url = "https://files.pythonhosted.org/packages/ad/0c/c804f5f922a9a6563bab712d8dcc70251e8af811fce4524d57c2c0fd49a4/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c", size = 736829 }, + { url = "https://files.pythonhosted.org/packages/51/16/6af8d6a6b210c8e54f1406a6b9481febf9c64a3109c541567e35a49aa2e7/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317", size = 764167 }, + { url = "https://files.pythonhosted.org/packages/75/e4/2c27590dfc9992f73aabbeb9241ae20220bd9452df27483b6e56d3975cc5/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85", size = 762952 }, + { url = "https://files.pythonhosted.org/packages/9b/97/ecc1abf4a823f5ac61941a9c00fe501b02ac3ab0e373c3857f7d4b83e2b6/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4", size = 735301 }, + { url = "https://files.pythonhosted.org/packages/45/73/0f49dacd6e82c9430e46f4a027baa4ca205e8b0a9dce1397f44edc23559d/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e", size = 756638 }, + { url = "https://files.pythonhosted.org/packages/22/5f/956f0f9fc65223a58fbc14459bf34b4cc48dec52e00535c79b8db361aabd/PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5", size = 143850 }, + { url = "https://files.pythonhosted.org/packages/ed/23/8da0bbe2ab9dcdd11f4f4557ccaf95c10b9811b13ecced089d43ce59c3c8/PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44", size = 161980 }, + { url = "https://files.pythonhosted.org/packages/86/0c/c581167fc46d6d6d7ddcfb8c843a4de25bdd27e4466938109ca68492292c/PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab", size = 183873 }, + { url = "https://files.pythonhosted.org/packages/a8/0c/38374f5bb272c051e2a69281d71cba6fdb983413e6758b84482905e29a5d/PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725", size = 173302 }, + { url = "https://files.pythonhosted.org/packages/c3/93/9916574aa8c00aa06bbac729972eb1071d002b8e158bd0e83a3b9a20a1f7/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5", size = 739154 }, + { url = "https://files.pythonhosted.org/packages/95/0f/b8938f1cbd09739c6da569d172531567dbcc9789e0029aa070856f123984/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425", size = 766223 }, + { url = "https://files.pythonhosted.org/packages/b9/2b/614b4752f2e127db5cc206abc23a8c19678e92b23c3db30fc86ab731d3bd/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476", size = 767542 }, + { url = "https://files.pythonhosted.org/packages/d4/00/dd137d5bcc7efea1836d6264f049359861cf548469d18da90cd8216cf05f/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48", size = 731164 }, + { url = "https://files.pythonhosted.org/packages/c9/1f/4f998c900485e5c0ef43838363ba4a9723ac0ad73a9dc42068b12aaba4e4/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b", size = 756611 }, + { url = "https://files.pythonhosted.org/packages/df/d1/f5a275fdb252768b7a11ec63585bc38d0e87c9e05668a139fea92b80634c/PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4", size = 140591 }, + { url = "https://files.pythonhosted.org/packages/0c/e8/4f648c598b17c3d06e8753d7d13d57542b30d56e6c2dedf9c331ae56312e/PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", size = 156338 }, + { url = "https://files.pythonhosted.org/packages/ef/e3/3af305b830494fa85d95f6d95ef7fa73f2ee1cc8ef5b495c7c3269fb835f/PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba", size = 181309 }, + { url = "https://files.pythonhosted.org/packages/45/9f/3b1c20a0b7a3200524eb0076cc027a970d320bd3a6592873c85c92a08731/PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1", size = 171679 }, + { url = "https://files.pythonhosted.org/packages/7c/9a/337322f27005c33bcb656c655fa78325b730324c78620e8328ae28b64d0c/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133", size = 733428 }, + { url = "https://files.pythonhosted.org/packages/a3/69/864fbe19e6c18ea3cc196cbe5d392175b4cf3d5d0ac1403ec3f2d237ebb5/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484", size = 763361 }, + { url = "https://files.pythonhosted.org/packages/04/24/b7721e4845c2f162d26f50521b825fb061bc0a5afcf9a386840f23ea19fa/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5", size = 759523 }, + { url = "https://files.pythonhosted.org/packages/2b/b2/e3234f59ba06559c6ff63c4e10baea10e5e7df868092bf9ab40e5b9c56b6/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc", size = 726660 }, + { url = "https://files.pythonhosted.org/packages/fe/0f/25911a9f080464c59fab9027482f822b86bf0608957a5fcc6eaac85aa515/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652", size = 751597 }, + { url = "https://files.pythonhosted.org/packages/14/0d/e2c3b43bbce3cf6bd97c840b46088a3031085179e596d4929729d8d68270/PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183", size = 140527 }, + { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446 }, +] + +[[package]] +name = "ruff" +version = "0.7.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/51/231bb3790e5b0b9fd4131f9a231d73d061b3667522e3f406fd9b63334d0e/ruff-0.7.2.tar.gz", hash = "sha256:2b14e77293380e475b4e3a7a368e14549288ed2931fce259a6f99978669e844f", size = 3210036 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/56/0caa2b5745d66a39aa239c01059f6918fc76ed8380033d2f44bf297d141d/ruff-0.7.2-py3-none-linux_armv6l.whl", hash = "sha256:b73f873b5f52092e63ed540adefc3c36f1f803790ecf2590e1df8bf0a9f72cb8", size = 10373973 }, + { url = "https://files.pythonhosted.org/packages/1a/33/cad6ff306731f335d481c50caa155b69a286d5b388e87ff234cd2a4b3557/ruff-0.7.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:5b813ef26db1015953daf476202585512afd6a6862a02cde63f3bafb53d0b2d4", size = 10171140 }, + { url = "https://files.pythonhosted.org/packages/97/f5/6a2ca5c9ba416226eac9cf8121a1baa6f06655431937e85f38ffcb9d0d01/ruff-0.7.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:853277dbd9675810c6826dad7a428d52a11760744508340e66bf46f8be9701d9", size = 9809333 }, + { url = "https://files.pythonhosted.org/packages/16/83/e3e87f13d1a1dc205713632978cd7bc287a59b08bc95780dbe359b9aefcb/ruff-0.7.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21aae53ab1490a52bf4e3bf520c10ce120987b047c494cacf4edad0ba0888da2", size = 10622987 }, + { url = "https://files.pythonhosted.org/packages/22/16/97ccab194480e99a2e3c77ae132b3eebfa38c2112747570c403a4a13ba3a/ruff-0.7.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ccc7e0fc6e0cb3168443eeadb6445285abaae75142ee22b2b72c27d790ab60ba", size = 10184640 }, + { url = "https://files.pythonhosted.org/packages/97/1b/82ff05441b036f68817296c14f24da47c591cb27acfda473ee571a5651ac/ruff-0.7.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd77877a4e43b3a98e5ef4715ba3862105e299af0c48942cc6d51ba3d97dc859", size = 11210203 }, + { url = "https://files.pythonhosted.org/packages/a6/96/7ecb30a7ef7f942e2d8e0287ad4c1957dddc6c5097af4978c27cfc334f97/ruff-0.7.2-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:e00163fb897d35523c70d71a46fbaa43bf7bf9af0f4534c53ea5b96b2e03397b", size = 11870894 }, + { url = "https://files.pythonhosted.org/packages/06/6a/c716bb126218227f8e604a9c484836257708a05ee3d2ebceb666ff3d3867/ruff-0.7.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f3c54b538633482dc342e9b634d91168fe8cc56b30a4b4f99287f4e339103e88", size = 11449533 }, + { url = "https://files.pythonhosted.org/packages/e6/2f/3a5f9f9478904e5ae9506ea699109070ead1e79aac041e872cbaad8a7458/ruff-0.7.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7b792468e9804a204be221b14257566669d1db5c00d6bb335996e5cd7004ba80", size = 12607919 }, + { url = "https://files.pythonhosted.org/packages/a0/57/4642e57484d80d274750dcc872ea66655bbd7e66e986fede31e1865b463d/ruff-0.7.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dba53ed84ac19ae4bfb4ea4bf0172550a2285fa27fbb13e3746f04c80f7fa088", size = 11016915 }, + { url = "https://files.pythonhosted.org/packages/4d/6d/59be6680abee34c22296ae3f46b2a3b91662b8b18ab0bf388b5eb1355c97/ruff-0.7.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:b19fafe261bf741bca2764c14cbb4ee1819b67adb63ebc2db6401dcd652e3748", size = 10625424 }, + { url = "https://files.pythonhosted.org/packages/82/e7/f6a643683354c9bc7879d2f228ee0324fea66d253de49273a0814fba1927/ruff-0.7.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:28bd8220f4d8f79d590db9e2f6a0674f75ddbc3847277dd44ac1f8d30684b828", size = 10233692 }, + { url = "https://files.pythonhosted.org/packages/d7/48/b4e02fc835cd7ed1ee7318d9c53e48bcf6b66301f55925a7dcb920e45532/ruff-0.7.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9fd67094e77efbea932e62b5d2483006154794040abb3a5072e659096415ae1e", size = 10751825 }, + { url = "https://files.pythonhosted.org/packages/1e/06/6c5ee6ab7bb4cbad9e8bb9b2dd0d818c759c90c1c9e057c6ed70334b97f4/ruff-0.7.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:576305393998b7bd6c46018f8104ea3a9cb3fa7908c21d8580e3274a3b04b691", size = 11074811 }, + { url = "https://files.pythonhosted.org/packages/a1/16/8969304f25bcd0e4af1778342e63b715e91db8a2dbb51807acd858cba915/ruff-0.7.2-py3-none-win32.whl", hash = "sha256:fa993cfc9f0ff11187e82de874dfc3611df80852540331bc85c75809c93253a8", size = 8650268 }, + { url = "https://files.pythonhosted.org/packages/d9/18/c4b00d161def43fe5968e959039c8f6ce60dca762cec4a34e4e83a4210a0/ruff-0.7.2-py3-none-win_amd64.whl", hash = "sha256:dd8800cbe0254e06b8fec585e97554047fb82c894973f7ff18558eee33d1cb88", size = 9433693 }, + { url = "https://files.pythonhosted.org/packages/7f/7b/c920673ac01c19814dd15fc617c02301c522f3d6812ca2024f4588ed4549/ruff-0.7.2-py3-none-win_arm64.whl", hash = "sha256:bb8368cd45bba3f57bb29cbb8d64b4a33f8415d0149d2655c5c8539452ce7760", size = 8735845 }, +]