#4293 Fix (?) do_multilib
Opened 2 months ago by figless. Modified 2 months ago
figless/koji fix_domultilib  into  master

file modified
+9 -96
@@ -6429,6 +6429,7 @@ 

          dnfbase = dnf.Base()

          mlm = multilib.DevelMultilibMethod(ml_conf)

          fs_missing = set()

+         ml_packages = {}

          for bnp in self.kojipkgs:

              rpminfo = self.kojipkgs[bnp]

              ppath = rpminfo['_pkgpath']
@@ -6437,7 +6438,7 @@ 

              if mlm.select(po):

                  # we need a multilib package to be included

                  ml_bnp = bnp.replace(arch, self.archmap[arch])

-                 ml_path = os.path.join(mldir, ml_bnp[0].lower(), ml_bnp)

+                 ml_path = os.path.join(mldir, 'Packages', ml_bnp[0].lower(), ml_bnp)

                  # ^ XXX - should actually generate this

                  if ml_bnp not in ml_pkgs:

                      # not in our multilib repo
@@ -6445,101 +6446,13 @@ 

                      fs_missing.add(ml_path)

                      # we defer failure so can report all the missing deps

                      continue

-                 ml_true.add(ml_path)

- 

-         # step 2: set up architectures for dnf configuration

-         self.logger.info("Resolving multilib for %s using method devel" % arch)

-         dnfdir = os.path.join(self.workdir, 'dnf')

-         # TODO: unwind this arch mess

-         archlist = (arch, 'noarch')

-         transaction_arch = arch

-         archlist = archlist + self.compat[self.biarch[arch]]

-         best_compat = self.compat[self.biarch[arch]][0]

-         if koji.arch.archDifference(best_compat, arch) > 0:

-             transaction_arch = best_compat

-         dnfconfig = """

- [main]

- debuglevel=2

- #pkgpolicy=newest

- #exactarch=1

- gpgcheck=0

- #reposdir=/dev/null

- #cachedir=/dnfcache

- installroot=%s

- #logfile=/dnf.log

- 

- [koji-%s]

- name=koji multilib task

- baseurl=file://%s

- enabled=1

- 

- """ % (dnfdir, self.id, mldir)

-         os.makedirs(os.path.join(dnfdir, "dnfcache"))

-         os.makedirs(os.path.join(dnfdir, 'var/lib/rpm'))

- 

-         # step 3: proceed with dnf config and set up

-         yconfig_path = os.path.join(dnfdir, 'dnf.conf-koji-%s' % arch)

-         with koji._open_text_file(yconfig_path, 'wt') as f:

-             f.write(dnfconfig)

-         self.session.uploadWrapper(yconfig_path, self.uploadpath,

-                                    os.path.basename(yconfig_path))

-         conf = dnf.conf.Conf()

-         conf.reposdir = []  # don't use system repos at all

-         conf.read(yconfig_path)

-         dnfbase = dnf.Base(conf)

-         if hasattr(koji.arch, 'ArchStorage'):

-             dnfbase.conf.arch = transaction_arch

-         else:

-             koji.arch.canonArch = transaction_arch

-         dnfbase.read_all_repos()

-         dnfbase.fill_sack(load_system_repo=False, load_available_repos=True)

-         for pkg in ml_true:

-             dnfbase.install(pkg)

- 

-         # step 4: execute dnf transaction to get dependencies

-         self.logger.info("Resolving dependencies for arch %s" % arch)

- 

-         ml_needed = {}

-         try:

-             dnfbase.resolve()

-             self.logger.info('dnf depsolve successfully finished')

-             for po in dnfbase.transaction.install_set:

-                 bnp = os.path.basename(po.localPkg())

-                 dep_path = os.path.join(mldir, bnp[0].lower(), bnp)

-                 ml_needed[dep_path] = po

-                 if not os.path.exists(dep_path):

-                     self.logger.error('%s (multilib dep) not on filesystem' % dep_path)

-                     fs_missing.add(dep_path)

-         except dnf.exceptions.DepsolveError:

-             self.logger.error('dnf depsolve was unsuccessful')

-             raise

- 

-         if len(fs_missing) > 0:

-             missing_log = os.path.join(self.workdir, 'missing_multilib.log')

-             with koji._open_text_file(missing_log, 'wt') as outfile:

-                 outfile.write('The following multilib files were missing:\n')

-                 for ml_path in fs_missing:

-                     outfile.write(ml_path + '\n')

-             self.session.uploadWrapper(missing_log, self.uploadpath)

-             raise koji.GenericError('multilib packages missing. '

-                                     'See missing_multilib.log')

- 

-         # step 5: update kojipkgs

-         for dep_path in ml_needed:

-             tspkg = ml_needed[dep_path]

-             bnp = os.path.basename(dep_path)

-             if bnp in self.kojipkgs:

-                 # we expect duplication with noarch, but not other arches

-                 if tspkg.arch != 'noarch':

-                     self.logger.warning("Multilib duplicate: %s", bnp)

-                 continue

-             rpminfo = ml_pkgs[bnp].copy()

-             # fix _pkgpath, which comes from another task and could be wrong

-             # for us

-             # TODO: would be better if we could use the proper path here

-             rpminfo['_pkgpath'] = dep_path

-             rpminfo['_multilib'] = True

-             self.kojipkgs[bnp] = rpminfo

+                 self.logger.info(f'{ml_path} should be added as a multilib rpm')

+                 ml_rpm = ml_pkgs[bnp.replace("x86_64", "i686")].copy()

+                 ml_bnp = os.path.basename(ml_path)

+                 ml_rpm['_pkgpath'] = ml_path

+                 ml_rpm['_multilib'] = True

+                 ml_packages[ml_bnp] = ml_rpm

+         self.kojipkgs.update(ml_packages)

  

      def pick_key(self, keys, avail_keys):

          best = None

Hello,

We have a need to generate el7 multilib repositories via koji dist-repo.

I couldn't get multilib (koji dist-repo ... --multilib multilib.conf) to work. Looking at the code, the first obvious problem was that the ml_path was incorrect as 'Packages' was not being included in the string.
After fixing this, I was then getting stuck as koji was trying to do a depsolve - which I don't understand why?
In the end I removed most of the logic in the existing do_multilib function and have now managed to have multilib dist-repo generation working for our use-case. That being said, I have no idea if it's "correct".

Is multilib generation still supported?
Does this PR make any sense?

Yep, it is broken. I've created #2539 for that but since no-one complained that it is not working I was quietly assuming that it is really not used anymore.

So, first - why is depsolving needed. You will not get transitive dependencies otherwise. So, you then rely on explicit whitelist in multilib.conf.

Second - there are more issues - your pr assumes i686/x86_64 which is not working for ppc/ppc64, and even not for i386/x86_64, etc.

If it is worth fixing (at least there is some user of this feature :-)) I would fix all of the problems at once. @mikem ?

I know we have folks using dist repos. I don't know the full extent of it.

Originally, dist-repos were intended to be a tool to facilitate faster composes (which is why a lot of the code is the way it is), but I don't think that's been a thing for a long time.

In Brew, we see a few dist-repos getting generated, but not often (and not with multilib). It doesn't look to me like there is any significant use.

In Fedora, they have the tag2distrepo plugin enabled (and oof, that plugin needs an overhaul). It seems that the flatpak plugin relies on it. However, the tag2distrepo plugin always sets multilib=False.

It seems to me that at least some uses of dist-repos are misuses that could be handled with regular repos.

We currently don't have/use any multilib dist-repos (well, clearly since it was broken), but there's a use case where we might want it (making openh264 repos)

It appears that the koji-flatpak plugin relies on dist repos, at least to some extent. @otaylor do you recall why those are desired there instead of normal repos?

It appears that the koji-flatpak plugin relies on dist repos, at least to some extent. @otaylor do you recall why those are desired there instead of normal repos?

IIRC, the reason is that we need the f41-flatpak-app tag to inherit from f41 (so that we inherit the packages from f41), but we need a repository that does not include inherited packages:

$ koji taginfo f41-flatpak-app
Tag: f41-flatpak-app [83545]
Arches: x86_64 aarch64 ppc64le
Groups: appliance-build, build, kiwi-build, livecd-build, livemedia-build, srpm-build
Tag options:
  mock.new_chroot : 1          [f41]
  mock.package_manager : 'dnf5' [f41]
  tag2distrepo.enabled : True
  tag2distrepo.inherit : False
  tag2distrepo.latest : True
Targets that build into this tag:
  f41-flatpak-app (f41-flatpak-app-build, repo#6544359: 2025-01-20 14:51:12.430614+00:00)
  f41-flatpak-runtime-kojira (f41-flatpak-runtime-packages, repo#6544453: 2025-01-21 02:53:07.601056+00:00)
Current repo: no active repo
Inheritance:
  0    ..I. f41 [83496]

cc: @yselkowitz

we need a repository that does not include inherited packages:

That's the same reason why we need dist repos.

Metadata