AOSC 2025 Python Survey

acbs.diff

diff --git a/acbs/base.py b/acbs/base.py
index fef5b4981ae3..8eb1e1837d02 100644
--- a/acbs/base.py
+++ b/acbs/base.py
@@ -47,15 +47,20 @@ class ACBSPackageInfo(object):
         # extra exported variables from spec
         self.exported: Dict[str, str] = {}
         # modifiers to be applied to the source file/folder (only available in autobuild4)
-        self.modifiers: str = ''
+        self.modifiers: str = ""
+
+    def to_build_spec(self) -> str:
+        if self.modifiers != "":
+            return f"{self.name}:{self.modifiers}"
+        else:
+            return self.name
 
     @staticmethod
     def is_in_stage2(modifiers: str) -> bool:
         return '+stage2' in modifiers.lower()
 
-
     def __repr__(self) -> str:
-        return f'<ACBSPackageInfo {self.name}: deps: {self.deps} ; uri: {self.source_uri}>'
+        return f'<ACBSPackageInfo {self.to_build_spec()}: deps: {self.deps} ; uri: {self.source_uri}>'
 
 
 class ACBSShrinkWrap(object):
diff --git a/acbs/checkpoint.py b/acbs/checkpoint.py
index 392bf30f9e4c..fdae5dc2e019 100644
--- a/acbs/checkpoint.py
+++ b/acbs/checkpoint.py
@@ -36,7 +36,7 @@ def checkpoint_dpkg() -> str:
 
 
 def checkpoint_text(packages: List[ACBSPackageInfo]) -> str:
-    return '\n'.join([package.name for package in packages])
+    return '\n'.join([package.to_build_spec() for package in packages])
 
 
 def checkpoint_to_group(packages: List[ACBSPackageInfo], path: str) -> str:
diff --git a/acbs/deps.py b/acbs/deps.py
index 409f482b0d46..ddd481c1632f 100644
--- a/acbs/deps.py
+++ b/acbs/deps.py
@@ -1,8 +1,11 @@
 from collections import OrderedDict, defaultdict, deque
+import os
+import logging
 from typing import Deque, Dict, List
 
 from acbs.find import find_package
 from acbs.parser import ACBSPackageInfo, check_buildability
+from acbs.pm import check_if_available
 
 # package information cache
 pool: Dict[str, ACBSPackageInfo] = {}
@@ -34,7 +37,7 @@ def prepare_for_reorder(package: ACBSPackageInfo, packages_list: List[str]) -> A
     new_installables = []
     for d in package.installables:
         # skip self-dependency
-        if d == package.name:
+        if d == package.name and check_if_available(d):
             new_installables.append(d)
             continue
         try:
@@ -56,19 +59,20 @@ def strongly_connected(search_path: str, packages_list: List[str], results: list
 
     # search package begin
     print(f'[{len(results) + 1}/{len(pool)}] {vert:30}\r', end='', flush=True)
+    stage2 = stage2 or (':+stage2' in vert)
     current_package = packages.get(vert)
     if current_package is None:
-        package = pool.get(vert) or find_package(vert, search_path, '+stage2' if stage2 else '')
+        package = pool.get(vert) or find_package(vert.split(':', 2)[0], search_path, '+stage2' if stage2 else '')
         if not package:
             raise ValueError(
                 f'Package {vert} not found')
         if isinstance(package, list):
             for s in package:
-                if vert == s.name:
+                if vert == s.to_build_spec():
                     current_package = s
-                    pool[s.name] = s
+                    pool[s.to_build_spec()] = s
                     continue
-                pool[s.name] = s
+                pool[s.to_build_spec()] = s
                 packages_list.append(s.name)
         else:
             current_package = package
@@ -88,6 +92,18 @@ def strongly_connected(search_path: str, packages_list: List[str], results: list
             lowlink[vert] = min(lowlink[p], lowlink[vert])
         # adjacent package is in the stack which means it is part of a loop
         elif stackstate[p] is True:
+            if (
+                p in packages_list
+                and (p != current_package.name or not stage2)
+                and ((not check_if_available(p)) or (f"{p}:+stage2" in packages_list))
+                and stackstate[f"{p}:+stage2"] is False
+                and os.path.exists(os.path.join(current_package.script_location, "defines.stage2"))
+            ):
+                logging.info(f"{vert}: searching stage2 for {p}")
+                p = f"{p}:+stage2"
+                if index[p] == -1:
+                    strongly_connected(search_path, packages_list, results, packages,
+                                    p, lowlink, index, stackstate, stack, True, depth)
             lowlink[vert] = min(lowlink[p], index[vert])
 
     w = ''
@@ -98,6 +114,14 @@ def strongly_connected(search_path: str, packages_list: List[str], results: list
         # if the stack only contains one vertex, then there is no loop there
         while w != vert:
             w = stack.pop()
-            result.append(pool[w])
+            p = pool[w]
+            if f"{w}:+stage2" in pool:
+                logging.info(f"{vert}: checking stage2 for {w}")
+                l = f"{w}:+stage2"
+                if lowlink[l] == index[l]:
+                    logging.info(f"{vert}: using stage2 for {w}")
+                    results.append(result)
+                    result = []
+            result.append(p)
             stackstate[w] = False
         results.append(result)
diff --git a/acbs/main.py b/acbs/main.py
index dcbd9934e768..14ea8d1b7330 100644
--- a/acbs/main.py
+++ b/acbs/main.py
@@ -19,7 +19,7 @@ from acbs.deps import prepare_for_reorder, tarjan_search
 from acbs.fetch import fetch_source, process_source
 from acbs.find import check_package_groups, find_package
 from acbs.parser import check_buildability, get_deps_graph, get_tree_by_name
-from acbs.pm import install_from_repo
+from acbs.pm import install_from_repo, installed_cache, available_cache
 from acbs.utils import (
     ACBSLogFormatter,
     ACBSLogPlainFormatter,
@@ -185,7 +185,18 @@ class BuildCore(object):
         # begin finding and resolving dependencies
         logging.info('Searching and resolving dependencies...')
         acbs.pm.reorder_mode = self.reorder
+        for i in self.build_queue:
+            if i.startswith("!"):
+                installed_cache[i[1:]] = False
+                available_cache[i[1:]] = False
+            else:
+                i, modifiers = self.strip_modifiers(i)
+                if ACBSPackageInfo.is_in_stage2(modifiers):
+                    installed_cache[i] = False
+                    available_cache[i] = False
         for n, i in enumerate(self.build_queue):
+            if i.startswith('!'):
+                continue
             i, modifiers = self.strip_modifiers(i)
             if not validate_package_name(i):
                 raise ValueError(f'Invalid package name: `{i}`')
@@ -231,7 +242,7 @@ class BuildCore(object):
         package_names = [p.name for p in packages]
         for pkg in packages:
             # prepare for re-order if necessary
-            logging.debug(f'Prepare for re-ordering: {pkg.name}')
+            logging.debug(f'Prepare for re-ordering: {pkg.to_build_spec()}')
             new_packages.append(prepare_for_reorder(pkg, package_names))
         graph = get_deps_graph(new_packages)
         return tarjan_search(graph, self.tree_dir, stage2)
@@ -272,7 +283,7 @@ class BuildCore(object):
         packages.clear()  # clear package list for the search results
         # here we will check if there is any loop in the dependency graph
         for dep in resolved:
-            if len(dep) > 1 or dep[0].name in dep[0].deps:
+            if len(dep) > 1:
                 # this is a SCC, aka a loop
                 logging.error('Found a loop in the dependency graph: {}'.format(
                     print_package_names(dep)))
@@ -293,6 +304,15 @@ class BuildCore(object):
         if not self.reorder:
             # TODO: correctly hoist the packages inside the groups
             check_package_groups(packages)
+        dirty_packages = set()
+        for p in packages:
+            if ACBSPackageInfo.is_in_stage2(p.modifiers):
+                dirty_packages.add(p.name)
+            elif p.name in dirty_packages:
+                dirty_packages.remove(p.name)
+        for p in dirty_packages:
+            logging.info(f"appending {p} stage3")
+            resolved.append(find_package(p, self.tree_dir, '', self.tmp_dir))
         return resolved
 
     def build_sequential(self, build_timings, packages: List[ACBSPackageInfo]):
diff --git a/acbs/parser.py b/acbs/parser.py
index f2acfb54b2bc..b795f24fc701 100644
--- a/acbs/parser.py
+++ b/acbs/parser.py
@@ -190,7 +190,7 @@ def get_deps_graph(packages: List[ACBSPackageInfo]) -> 'OrderedDict[str, ACBSPac
     'convert flattened list to adjacency list'
     result = {}
     for i in packages:
-        result[i.name] = i
+        result[i.to_build_spec()] = i
     return OrderedDict(result)
 
 
diff --git a/acbs/pm.py b/acbs/pm.py
index b522d9c04f73..b34630f25ca7 100644
--- a/acbs/pm.py
+++ b/acbs/pm.py
@@ -113,10 +113,11 @@ def check_if_available(name: str) -> bool:
             ['apt-cache', 'show', escape_package_name(name)], stderr=subprocess.STDOUT)
         logging.debug('Checking if %s can be installed' % name)
         subprocess.check_output(
-            ['apt-get', 'install', '-s', name], stderr=subprocess.STDOUT, env={'DEBIAN_FRONTEND': 'noninteractive'})
+            ['apt-get', 'install', '-s', '--no-remove', name], stderr=subprocess.STDOUT, env={'DEBIAN_FRONTEND': 'noninteractive'})
         available_cache[name] = True
         return True
     except subprocess.CalledProcessError:
+        logging.info(f"Package {name} is not available")
         available_cache[name] = False
         return False
 
diff --git a/acbs/utils.py b/acbs/utils.py
index de48dc93b2d3..ef3bf8fc5810 100644
--- a/acbs/utils.py
+++ b/acbs/utils.py
@@ -135,7 +135,7 @@ def print_package_names(packages: List[ACBSPackageInfo], limit: Optional[int] =
     pkgs = packages
     if limit is not None and len(packages) > limit:
         pkgs = packages[:limit]
-    printable_packages = [pkg.name for pkg in pkgs]
+    printable_packages = [pkg.to_build_spec() for pkg in pkgs]
     more_messages = ' ... and {} more'.format(
         len(packages) - limit) if limit and limit < len(packages) else ''
     return ', '.join(printable_packages) + more_messages

acbs.sh

dpkg -i /debs/p/python-3_*.deb

# Removing acbs and pip to fix ACBS bootstrapping.
dpkg --force-all --purge acbs pip
apt install autobuild4

git clone https://github.com/AOSC-Dev/acbs --depth 1
export HOME=/root
export PATH="$HOME/.local/bin:$PATH"
cd acbs

# This step also installs get-pip.py, a key step that also introduces an updated setuptools.
# Without an update setuptools, setuptools-python3 will fail to build.
# Ignore the last error of /var/lib/acbs/repo not found
# bootstrap script uses setup.py but setuptools is not installed
# so install it manually
./bootstrap || true
pip install --user setuptools setuptools-scm
./bootstrap || true

# Replacing copy of ACBS in system root.
python3 setup.py install

# Generating a new ACBS configuration file.
cat > /etc/acbs/forest.conf << EOF
[default]
location = /tree
EOF

# Build dependencies to use PEP 517 autobuild template
# Bootstrap from pip
pip install build packaging flit-core installer hatchling hatch-vcs editables
acbs-build tomli pyproject-hooks python-build pep517 python-installer flit-core packaging hatchling hatch-vcs
# Build setuptools and replace user-directory installation.
acbs-build setuptools-python3:+stage2 setuptools setuptools-scm
pip uninstall -y setuptools setuptools-scm
acbs-build setuptools-python3 wheel pip virtualenv

# Install acbs dependencies
pip install jaraco.text platformdirs

acbs-build file pexpect pyparsing pycryptodome ptyprocess acbs

acbs-build pathspec pluggy calver trove-classifiers

build.py

#!/usr/bin/python3

# To enable TMPFS:
# ciel rollback
# ciel down
# ciel del py3
# mkdir .ciel/container/instances/py3
# mount -t tmpfs -o size=64G tmpfs .ciel/container/instances/py3
# ciel add py3

# To resize TMPFS (for LLVM, use 128G):
# mount -t tmpfs -o size=64G,remount tmpfs .ciel/container/instances/py3
# mount -t tmpfs -o size=128G,remount tmpfs .ciel/container/instances/py3

# To disable TMPFS:
# ciel rollback
# ciel down
# ciel del py3
# umount .ciel/container/instances/py3
# ciel add py3

import os
import subprocess
import sys


updateScript = """
set -euo pipefail
echo 'deb [trusted=yes] file:///debs/ /' >/etc/apt/sources.list.d/ciel-local.list
export OMA_NO_BELL=1 OMA_NO_PROGRESS=1
oma upgrade -y --force-confnew --force-unsafe-io
oma autoremove -y --remove-config
rm /etc/apt/sources.list.d/ciel-local.list
"""

checkRebuildScript = """
set -euo pipefail
echo 'deb [trusted=yes] file:///debs/ /' >/etc/apt/sources.list.d/ciel-local.list
oma refresh &>/dev/null
apt list '?any-version(?version(\\~pre.*Z))' 2>/dev/null | grep -v -F 'Listing ...' | grep -F '/stable' | grep -v '\-dbg/' || true
"""

outputDir = "OUTPUT-python-survey-2025"


def status(status):
    print(status, flush=True)
    with open("status", "w") as f:
        f.write(f"{status}\n")


def rebuild(pkg: str):
    if pkg in built:
        print(f"Requesting rebuild for {pkg}")
        built.remove(pkg)

        prefix = pkg[0] if not pkg.startswith("lib") else pkg[0:4]
        dir = f"debs/{prefix}"
        for file in os.listdir(f"{outputDir}/{dir}"):
            if file.endswith(".deb") and file.startswith(f"{pkg}_"):
                os.remove(f"{outputDir}/{dir}/{file}")
                print(f"rm {dir}/{file}")
    if pkg not in packages:
        print(f"{pkg} is not in the build plan")
        exit(1)


with open("group.txt", "r") as f:
    group = f.read().splitlines()
with open("built.txt", "a") as f:
    pass
with open("built.txt", "r") as f:
    built = set(f.read().splitlines())

packages = ["python-3:-pkgbreak"]
packages.extend(group)
packages.append("python-3")

print(f"All {len(packages)} packages")

for pkg in sys.argv[1:]:
    rebuild(pkg)

output = (
    subprocess.check_output(["ciel", "shell", "-i", "py3", checkRebuildScript])
    .decode()
    .strip()
)
outdatedPackages = [line.split("/")[0].strip() for line in output.split("\n")]
if len(output) != 0:
    status(f"Some packages are outdated:\n{output}")
    if "acbs" in outdatedPackages:
        exit(1)
    for pkg in outdatedPackages:
        rebuild(pkg)

for i, pkg in enumerate(packages):
    if pkg in built:
        continue
    status(f"[{i}/{len(packages)}] {pkg}")
    ret = subprocess.call(["ciel", "build", "-i", "py3", pkg])
    if ret != 0:
        status(f"[{i}/{len(packages)}] {pkg} FAILED (exit code: {ret})")
        exit(ret)
    else:
        built.add(pkg)
        with open("built.txt", "w") as f:
            f.write("\n".join(built))
        print(f"[{i}/{len(packages)}] {pkg} SUCCEEDED")
        if pkg == "python-3:-pkgbreak":
            status("ATTENTION: Bootstrap ACBS manually")
            exit()
    if i % 30 == 0:
        status(f"{i}: updating system (rollback)")
        ret = subprocess.call(["ciel", "rollback"])
        if ret != 0:
            status("ATTENTION: Update failed")
            exit()
        status(f"{i}: updating system (add)")
        ret = subprocess.call(["ciel", "add", "update"])
        if ret != 0:
            status("ATTENTION: Update failed")
            exit()
        status(f"{i}: updating system (upgrade)")
        ret = subprocess.call(["ciel", "sh", "-i", "update", updateScript])
        if ret != 0:
            status("ATTENTION: Update failed")
            exit()
        status(f"{i}: updating system (commit)")
        ret = subprocess.call(["ciel", "commit", "-i", "update"])
        if ret != 0:
            status("ATTENTION: Update failed")
            exit()
        status(f"{i}: updating system (del)")
        ret = subprocess.call(["ciel", "del", "update"])
        if ret != 0:
            status("ATTENTION: Update failed")
            exit()

print("BUILD COMPLETED")

pysort

#!/usr/bin/bash
set -euo pipefail

args=(
    # Packages that are also changed in this topic
    "grpc"
    "swig"
    "vala"
    "docbook-sgml"
)

while read -r dir; do
    pkg="$(cut -d'/' -f2 <<<"$dir")"
    if [[ "$pkg" =~ ^(tomli|pyproject-hooks|python-build|pep517|python-installer|flit-core|packaging|hatchling|hatch-vcs)$ ]]; then
        echo "$pkg is bootstrapped from pypi"
        continue
    fi
    if [[ "$pkg" =~ ^(setuptools-scm|setuptools-python3|wheel|pip|virtualenv|acbs)$ ]]; then
        echo "$pkg is bootstrapped from pypi"
        continue
    fi
    if [[ "$pkg" =~ ^(pathspec|pluggy|calver|trove-classifiers)$ ]]; then
        echo "$pkg is bootstrapped from pypi"
        continue
    fi
    if [[ "$pkg" =~ ^(pygtk|kmod)$ ]]; then
        echo "$pkg does not need rebuild"
        continue
    fi
    if [[ "$pkg" =~ ^(llvm-18|glib|llvm)$ ]]; then
        echo "$pkg can be reused (explicit)"
        args+=("$pkg")
        continue
    fi
    a="$(
        set +u
        source "$dir/spec"
        source "$dir/autobuild/defines"
        echo -n "${DUMMYSRC:-0}${ABTYPE:-0}"
    )"
    if [[ "$a" == "1dummy" ]] && rg -i transition "$dir/autobuild/defines"; then
        echo "$pkg can be reused (auto)"
        args+=("$pkg")
        continue
    fi
    args+=("$pkg" "!$pkg")
done < <(tail -n+4 groups/py3-rebuild)

echo "Args: ${args[*]}"

../acbs/acbs-build -ep "${args[@]}"

mv groups/acbs-* ../py3/group.txt