98 lines
3.8 KiB
Python
98 lines
3.8 KiB
Python
import json
|
|
import os
|
|
import sys
|
|
|
|
# Pass vcpkg.json files to merge their dependencies and produce a single vcpkg.json with their
|
|
# combined & deduplicated dependencies. Note that this script is very dumb and some manual merging may be required
|
|
# to combine extensions from multiple builds in the case of colliding dependencies.
|
|
|
|
# Also: note that due to the fact that the httpfs extension currently can not use the latest openssl version (3.1),
|
|
# we need to pin the openssl version requiring us to also pin the vcpkg version here. When updating the vcpkg git hash
|
|
# we probably want to change it here and in ('.github/actions/build_extensions/action.yml') at the same time
|
|
|
|
dependencies_str = []
|
|
dependencies_dict = []
|
|
merged_overlay_ports = []
|
|
merged_overlay_triplets = []
|
|
|
|
|
|
def prefix_overlay_ports_or_triples(overlay_dir, path_to_vcpkg_json):
|
|
def prefix_overlay_port_or_triplet(overlay_port_or_triplet):
|
|
vcpkg_prefix_path = path_to_vcpkg_json[0 : path_to_vcpkg_json.find("/vcpkg.json")]
|
|
if len(vcpkg_prefix_path) == 0:
|
|
return overlay_port_or_triplet
|
|
return vcpkg_prefix_path + '/' + overlay_port_or_triplet
|
|
|
|
return map(prefix_overlay_port_or_triplet, overlay_dir)
|
|
|
|
|
|
for file in sys.argv[1:]:
|
|
f = open(file)
|
|
data = json.load(f)
|
|
|
|
if 'dependencies' in data:
|
|
for dep in data['dependencies']:
|
|
if type(dep) is str:
|
|
dependencies_str.append(dep)
|
|
elif type(dep) is dict:
|
|
dependencies_dict.append(dep)
|
|
else:
|
|
raise Exception(f"Unknown entry type found in dependencies: '{dep}'")
|
|
|
|
if 'vcpkg-configuration' in data:
|
|
if 'overlay-ports' in data['vcpkg-configuration']:
|
|
merged_overlay_ports += prefix_overlay_ports_or_triples(data['vcpkg-configuration']['overlay-ports'], file)
|
|
if 'overlay-triplets' in data['vcpkg-configuration']:
|
|
merged_overlay_triplets += prefix_overlay_ports_or_triples(
|
|
data['vcpkg-configuration']['overlay-triplets'], file
|
|
)
|
|
|
|
final_deduplicated_deps = list()
|
|
dedup_set = set()
|
|
|
|
for dep in dependencies_dict:
|
|
if dep['name'] not in dedup_set:
|
|
final_deduplicated_deps.append(dep)
|
|
# TODO: deduplication is disabled for now, just let vcpkg handle duplicates in deps
|
|
# dedup_set.add(dep['name'])
|
|
|
|
for dep in dependencies_str:
|
|
if dep not in dedup_set:
|
|
final_deduplicated_deps.append(dep)
|
|
# TODO: deduplication is disabled for now, just let vcpkg handle duplicates in deps
|
|
# dedup_set.add(dep)
|
|
|
|
opensslVersion = os.getenv("OPENSSL_VERSION_OVERRIDE", "3.0.8")
|
|
data = {
|
|
"description": f"Auto-generated vcpkg.json for combined DuckDB extension build, generated by 'scripts/merge_vcpkg_deps.py'",
|
|
"builtin-baseline": "ce613c41372b23b1f51333815feb3edd87ef8a8b",
|
|
"dependencies": final_deduplicated_deps,
|
|
"overrides": [{"name": "openssl", "version": opensslVersion}],
|
|
}
|
|
|
|
data['vcpkg-configuration'] = {}
|
|
|
|
if merged_overlay_ports:
|
|
data['vcpkg-configuration']['overlay-ports'] = merged_overlay_ports
|
|
|
|
if merged_overlay_triplets:
|
|
data['vcpkg-configuration']['overlay-triplets'] = merged_overlay_triplets
|
|
|
|
REGISTRY_BASELINE = '869bddccca976e0abe25894356e7f49e77765169'
|
|
# NOTE: use 'scripts/list_vcpkg_registry_packages.py --baseline <baseline>' to generate the list of packages
|
|
data['vcpkg-configuration']['registries'] = [
|
|
{
|
|
"kind": "git",
|
|
"repository": "https://github.com/duckdb/vcpkg-duckdb-ports",
|
|
"baseline": REGISTRY_BASELINE,
|
|
"packages": ['avro-c', 'vcpkg-cmake'],
|
|
}
|
|
]
|
|
|
|
# Print output
|
|
print("Writing to 'build/extension_configuration/vcpkg.json': ")
|
|
print(data["dependencies"])
|
|
|
|
with open('build/extension_configuration/vcpkg.json', 'w', encoding='utf-8') as f:
|
|
json.dump(data, f, ensure_ascii=False, indent=4)
|