should be it

external/duckdb/scripts/merge_vcpkg_deps.py

@@ -0,0 +1,97 @@
+import json
+import os
+import sys
+
+# Pass vcpkg.json files to merge their dependencies and produce a single vcpkg.json with their
+# combined & deduplicated dependencies. Note that this script is very dumb and some manual merging may be required
+# to combine extensions from multiple builds in the case of colliding dependencies.
+
+# Also: note that due to the fact that the httpfs extension currently can not use the latest openssl version (3.1),
+# we need to pin the openssl version requiring us to also pin the vcpkg version here. When updating the vcpkg git hash
+# we probably want to change it here and in ('.github/actions/build_extensions/action.yml') at the same time
+
+dependencies_str = []
+dependencies_dict = []
+merged_overlay_ports = []
+merged_overlay_triplets = []
+
+
+def prefix_overlay_ports_or_triples(overlay_dir, path_to_vcpkg_json):
+    def prefix_overlay_port_or_triplet(overlay_port_or_triplet):
+        vcpkg_prefix_path = path_to_vcpkg_json[0 : path_to_vcpkg_json.find("/vcpkg.json")]
+        if len(vcpkg_prefix_path) == 0:
+            return overlay_port_or_triplet
+        return vcpkg_prefix_path + '/' + overlay_port_or_triplet
+
+    return map(prefix_overlay_port_or_triplet, overlay_dir)
+
+
+for file in sys.argv[1:]:
+    f = open(file)
+    data = json.load(f)
+
+    if 'dependencies' in data:
+        for dep in data['dependencies']:
+            if type(dep) is str:
+                dependencies_str.append(dep)
+            elif type(dep) is dict:
+                dependencies_dict.append(dep)
+            else:
+                raise Exception(f"Unknown entry type found in dependencies: '{dep}'")
+
+    if 'vcpkg-configuration' in data:
+        if 'overlay-ports' in data['vcpkg-configuration']:
+            merged_overlay_ports += prefix_overlay_ports_or_triples(data['vcpkg-configuration']['overlay-ports'], file)
+        if 'overlay-triplets' in data['vcpkg-configuration']:
+            merged_overlay_triplets += prefix_overlay_ports_or_triples(
+                data['vcpkg-configuration']['overlay-triplets'], file
+            )
+
+final_deduplicated_deps = list()
+dedup_set = set()
+
+for dep in dependencies_dict:
+    if dep['name'] not in dedup_set:
+        final_deduplicated_deps.append(dep)
+        # TODO: deduplication is disabled for now, just let vcpkg handle duplicates in deps
+        # dedup_set.add(dep['name'])
+
+for dep in dependencies_str:
+    if dep not in dedup_set:
+        final_deduplicated_deps.append(dep)
+        # TODO: deduplication is disabled for now, just let vcpkg handle duplicates in deps
+        # dedup_set.add(dep)
+
+opensslVersion = os.getenv("OPENSSL_VERSION_OVERRIDE", "3.0.8")
+data = {
+    "description": f"Auto-generated vcpkg.json for combined DuckDB extension build, generated by 'scripts/merge_vcpkg_deps.py'",
+    "builtin-baseline": "ce613c41372b23b1f51333815feb3edd87ef8a8b",
+    "dependencies": final_deduplicated_deps,
+    "overrides": [{"name": "openssl", "version": opensslVersion}],
+}
+
+data['vcpkg-configuration'] = {}
+
+if merged_overlay_ports:
+    data['vcpkg-configuration']['overlay-ports'] = merged_overlay_ports
+
+if merged_overlay_triplets:
+    data['vcpkg-configuration']['overlay-triplets'] = merged_overlay_triplets
+
+REGISTRY_BASELINE = '869bddccca976e0abe25894356e7f49e77765169'
+# NOTE: use 'scripts/list_vcpkg_registry_packages.py --baseline <baseline>' to generate the list of packages
+data['vcpkg-configuration']['registries'] = [
+    {
+        "kind": "git",
+        "repository": "https://github.com/duckdb/vcpkg-duckdb-ports",
+        "baseline": REGISTRY_BASELINE,
+        "packages": ['avro-c', 'vcpkg-cmake'],
+    }
+]
+
+# Print output
+print("Writing to 'build/extension_configuration/vcpkg.json': ")
+print(data["dependencies"])
+
+with open('build/extension_configuration/vcpkg.json', 'w', encoding='utf-8') as f:
+    json.dump(data, f, ensure_ascii=False, indent=4)