import json import os import sys # Pass vcpkg.json files to merge their dependencies and produce a single vcpkg.json with their # combined & deduplicated dependencies. Note that this script is very dumb and some manual merging may be required # to combine extensions from multiple builds in the case of colliding dependencies. # Also: note that due to the fact that the httpfs extension currently can not use the latest openssl version (3.1), # we need to pin the openssl version requiring us to also pin the vcpkg version here. When updating the vcpkg git hash # we probably want to change it here and in ('.github/actions/build_extensions/action.yml') at the same time dependencies_str = [] dependencies_dict = [] merged_overlay_ports = [] merged_overlay_triplets = [] def prefix_overlay_ports_or_triples(overlay_dir, path_to_vcpkg_json): def prefix_overlay_port_or_triplet(overlay_port_or_triplet): vcpkg_prefix_path = path_to_vcpkg_json[0 : path_to_vcpkg_json.find("/vcpkg.json")] if len(vcpkg_prefix_path) == 0: return overlay_port_or_triplet return vcpkg_prefix_path + '/' + overlay_port_or_triplet return map(prefix_overlay_port_or_triplet, overlay_dir) for file in sys.argv[1:]: f = open(file) data = json.load(f) if 'dependencies' in data: for dep in data['dependencies']: if type(dep) is str: dependencies_str.append(dep) elif type(dep) is dict: dependencies_dict.append(dep) else: raise Exception(f"Unknown entry type found in dependencies: '{dep}'") if 'vcpkg-configuration' in data: if 'overlay-ports' in data['vcpkg-configuration']: merged_overlay_ports += prefix_overlay_ports_or_triples(data['vcpkg-configuration']['overlay-ports'], file) if 'overlay-triplets' in data['vcpkg-configuration']: merged_overlay_triplets += prefix_overlay_ports_or_triples( data['vcpkg-configuration']['overlay-triplets'], file ) final_deduplicated_deps = list() dedup_set = set() for dep in dependencies_dict: if dep['name'] not in dedup_set: final_deduplicated_deps.append(dep) # TODO: deduplication is disabled for now, just let vcpkg handle duplicates in deps # dedup_set.add(dep['name']) for dep in dependencies_str: if dep not in dedup_set: final_deduplicated_deps.append(dep) # TODO: deduplication is disabled for now, just let vcpkg handle duplicates in deps # dedup_set.add(dep) opensslVersion = os.getenv("OPENSSL_VERSION_OVERRIDE", "3.0.8") data = { "description": f"Auto-generated vcpkg.json for combined DuckDB extension build, generated by 'scripts/merge_vcpkg_deps.py'", "builtin-baseline": "ce613c41372b23b1f51333815feb3edd87ef8a8b", "dependencies": final_deduplicated_deps, "overrides": [{"name": "openssl", "version": opensslVersion}], } data['vcpkg-configuration'] = {} if merged_overlay_ports: data['vcpkg-configuration']['overlay-ports'] = merged_overlay_ports if merged_overlay_triplets: data['vcpkg-configuration']['overlay-triplets'] = merged_overlay_triplets REGISTRY_BASELINE = '869bddccca976e0abe25894356e7f49e77765169' # NOTE: use 'scripts/list_vcpkg_registry_packages.py --baseline ' to generate the list of packages data['vcpkg-configuration']['registries'] = [ { "kind": "git", "repository": "https://github.com/duckdb/vcpkg-duckdb-ports", "baseline": REGISTRY_BASELINE, "packages": ['avro-c', 'vcpkg-cmake'], } ] # Print output print("Writing to 'build/extension_configuration/vcpkg.json': ") print(data["dependencies"]) with open('build/extension_configuration/vcpkg.json', 'w', encoding='utf-8') as f: json.dump(data, f, ensure_ascii=False, indent=4)