Skip to content

Commit

Permalink
Merge pull request #1 from stdweird/yumdnf_spmaleaves
Browse files Browse the repository at this point in the history
Yumdnf spmaleaves
  • Loading branch information
StephaneGerardVUB committed Nov 30, 2023
2 parents 756e746 + 22c9bc6 commit 2835821
Show file tree
Hide file tree
Showing 6 changed files with 310 additions and 17 deletions.
59 changes: 59 additions & 0 deletions ncm-spma/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,25 @@
</resources>
</configuration>
</execution>
<execution>
<id>filter-share-spma-yumdnf</id>
<phase>process-sources</phase>
<goals>
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/share/quattor/spma/yumdnf</outputDirectory>
<resources>
<resource>
<directory>src/main/resources/yumdnf</directory>
<includes>
<include>*.py</include>
</includes>
<filtering>false</filtering>
</resource>
</resources>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
Expand Down Expand Up @@ -268,7 +287,47 @@
</source>
</sources>
</mapping>
<mapping>
<directory>/usr/share/quattor/spma/yumdnf</directory>
<directoryIncluded>false</directoryIncluded>
<sources>
<source>
<location>${project.build.directory}/share/quattor/spma/yumdnf</location>
<includes>
<include>*.py</include>
</includes>
</source>
</sources>
</mapping>
</mappings>
<postinstallScriptlet>
<script>
if [ -x /usr/bin/dnf ]; then
# from dnf.const.PLUGINPATH
pylib=$(/usr/libexec/platform-python -c 'import distutils.sysconfig;print(distutils.sysconfig.get_python_lib())')
plugin="$pylib/dnf-plugins/spmaleaves.py"
if [ ! -f $plugin ]; then
ln -s /usr/share/quattor/spma/yumdnf/spmaleaves.py $plugin
fi
fi
</script>
</postinstallScriptlet>
<preremoveScriptlet>
<script>
if [ $1 == 0 ]; then
if [ -x /usr/bin/dnf ]; then
# from dnf.const.PLUGINPATH
pylib=$(/usr/libexec/platform-python -c 'import distutils.sysconfig;print(distutils.sysconfig.get_python_lib())')
plugin="$pylib/dnf-plugins/spmaleaves.py"
if [ -f $plugin ]; then
rm -f $plugin
rm -f "$pylib"/dnf-plugins/__pycache__/spmaleaves.*.pyc
fi
fi
fi
</script>
</preremoveScriptlet>

</configuration>
</plugin>
</plugins>
Expand Down
44 changes: 28 additions & 16 deletions ncm-spma/src/main/perl/spma/yum.pm
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ use constant LARGE_INSTALL => 200;
use constant NOACTION_TEMPDIR_TEMPLATE => "/tmp/spma-noaction-XXXXX";

use constant YUM_CONF_CLEANUP_ON_REMOVE => "clean_requirements_on_remove";
use constant YUM_CONF_CLEANUP_ON_REMOVE_VALUE => 1;
use constant YUM_CONF_OBSOLETES => "obsoletes";
use constant YUM_CONF_PLUGINCONFPATH => 'pluginconfpath';
use constant YUM_CONF_REPOSDIR => 'reposdir';
Expand Down Expand Up @@ -328,16 +329,16 @@ sub execute_yum_command

$cmd->execute();
if ($err) {
# dnf always reports "Last metadata expiration check" to stderr, even if there's nothing wrong.
# If that is the only message, we should not consider it to be a warning, and instead report it as verbose.
# dnf always reports "Last metadata expiration check" to stderr, even if there's nothing wrong.
# If that is the only message, we should not consider it to be a warning, and instead report it as verbose.
my $warn_logger = ($err =~ m/\A[\s\n]*Last metadata expiration check.*[\s\n]*\z/m) ? 'verbose': 'warn';
$self->$warn_logger("$why produced warnings: $err");
}
$self->verbose("$why output: $out") if(defined($out));
if ($? ||
($err && $err =~ m{^\s*(
Error |
Failed |
Failed (?! (?: \s+ (?: loading \s+ plugin \s+))) |
(?:Could \s+ not \s+ match) |
(?:Transaction \s+ encountered .* error) |
(?:Unknown \s+ group \s+ package \s+ type) |
Expand Down Expand Up @@ -675,6 +676,25 @@ sub versionlock
return 1;
}

# return package to remove
# candidates are (non-trivial) values from the leaves selection command (name;arch format)
# wanted is list of packages that spma should install (name and also name;arch format)
sub _pkg_rem_calc
{
my ($self, $candidates, $wanted) = @_;

my $false_positives = Set::Scalar->new();
foreach my $pkg (@$candidates) {
my $name = (split(/;/, $pkg))[0];
if ($wanted->has($name)) {
$false_positives->insert($pkg);
}
}

return $candidates - $false_positives;
};


# Returns the set of packages to remove. A package must be removed if
# it is a leaf package and is not listed in $wanted, or if its
# architecture doesn't match the architectures specified in $wanted
Expand All @@ -683,7 +703,7 @@ sub packages_to_remove
{
my ($self, $wanted) = @_;

my $out = CAF::Process->new($self->_set_yum_config(LEAF_PACKAGES),
my $out = CAF::Process->new($self->_set_yum_config($self->LEAF_PACKAGES),
keeps_state => 1,
log => $self)->output();

Expand All @@ -696,17 +716,9 @@ sub packages_to_remove
# garbage.
my $leaves = Set::Scalar->new(grep($_ !~ m{\s}, split(/\n/, $out)));

my $candidates = $leaves-$wanted;

my $false_positives = Set::Scalar->new();
foreach my $pkg (@$candidates) {
my $name = (split(/;/, $pkg))[0];
if ($wanted->has($name)) {
$false_positives->insert($pkg);
}
}
my $candidates = $leaves - $wanted;

return $candidates-$false_positives;
return $self->_pkg_rem_calc($candidates, $wanted);
}

# Queries for packages packages that depend on $rm, and if there is a
Expand Down Expand Up @@ -880,7 +892,7 @@ sub update_pkgs
my $clean_cache_method = ($purge ? 'purge' : 'expire') . "_yum_caches";
$self->$clean_cache_method() or return 0;

$self->make_cache($purge) or return 0;
$self->make_cache() or return 0;
};

# Versionlock is determined based on all configured packages
Expand Down Expand Up @@ -1128,7 +1140,7 @@ sub configure_yum

# use CONSTANT() to get the value of the constant as key
my $yum_opts = {
YUM_CONF_CLEANUP_ON_REMOVE() => 1,
YUM_CONF_CLEANUP_ON_REMOVE() => $self->YUM_CONF_CLEANUP_ON_REMOVE_VALUE(),
YUM_CONF_OBSOLETES() => $obsoletes,
YUM_CONF_REPOSDIR() => $repodirs,
YUM_CONF_PLUGINCONFPATH() => $plugindir,
Expand Down
38 changes: 38 additions & 0 deletions ncm-spma/src/main/perl/spma/yumdnf.pm
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@ use warnings;
package NCM::Component::spma::yumdnf;

use NCM::Component::spma::yum;
use Set::Scalar;
push our @ISA , qw(NCM::Component::spma::yum);

use constant YUM_CONF_FILE => "/etc/dnf/dnf.conf";
use constant YUM_CONF_CLEANUP_ON_REMOVE_VALUE => 0;

use constant DNF_MODULES_DIR => "/etc/dnf/modules.d";

Expand All @@ -22,6 +24,42 @@ use constant REPO_INCLUDE => 0;

use constant MODULES_TREE => "/software/modules";

# package-cleanup --leaves really does not what it says. it doesn't even do orphans
use constant LEAF_PACKAGES => [qw(dnf spmaleaves)];

# return packages to remove
# candidates are (non-trivial) values from the leaves selection command (name;arch format)
# spmaleaves returns "strongly connected components" instead of individual packges
# so candidates might also be in the scc name1;arch1::name2;arch2 format
# wanted is list of packages that spma should install (name and also name;arch format)
sub _pkg_rem_calc
{
my ($self, $candidates, $wanted) = @_;

my $false_positives = Set::Scalar->new();
foreach my $scc (@$candidates) {
foreach my $pkg (split/::/, $scc) {
my $name = (split(/;/, $pkg))[0];
if ($wanted->has($pkg) || $wanted->has($name)) {
# the whole scc is flagged as soon as one component is wanted
$false_positives->insert($scc);
}
}
}

my $scc_remove = $candidates - $false_positives;

# Unpack the scc entries
my $remove = Set::Scalar->new();
foreach my $scc (@$scc_remove) {
foreach my $pkg (split/::/, $scc) {
$remove->insert($pkg);
};
}

return $remove;
};


# Completes any pending transactions
sub _do_complete_transaction
Expand Down
165 changes: 165 additions & 0 deletions ncm-spma/src/main/resources/yumdnf/spmaleaves.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
# leaves.py
# DNF plugin for listing installed packages not required by any other
# installed package.
#
# Copyright (C) 2015 Emil Renner Berthing
#
# This copyrighted material is made available to anyone wishing to use,
# modify, copy, or redistribute it subject to the terms and conditions of
# the GNU General Public License v.2, or (at your option) any later version.
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY expressed or implied, including the implied warranties of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# History:
# based on leaves.py dnf core plugin 84b7001
# Modifications:
# run method prints name;arch format
# rename classes and name/aliases
# patches of https://github.com/rpm-software-management/dnf-plugins-core/pull/399/ to return list of
# cyclic sets in custom format
import dnf
import dnf.sack
import dnf.cli
from dnfpluginscore import _


class SpmaLeaves(dnf.Plugin):
name = 'spmaleaves'

def __init__(self, base, cli):
super(SpmaLeaves, self).__init__(base, cli)
if cli:
cli.register_command(SpmaLeavesCommand)


class SpmaLeavesCommand(dnf.cli.Command):
aliases = ('spmaleaves',)
summary = _('List installed packages not required by any other package')

def buildgraph(self):
"""
Load the list of installed packages and their dependencies using
hawkey, and build the dependency graph and the graph of reverse
dependencies.
"""
query = dnf.sack._rpmdb_sack(self.base).query().apply()
pkgmap = dict()
packages = []
depends = []
rdepends = []
deps = set()
providers = set()

for i, pkg in enumerate(query):
pkgmap[pkg] = i
packages.append(pkg)
rdepends.append([])

for i, pkg in enumerate(packages):
for req in pkg.requires:
sreq = str(req)
if sreq.startswith('rpmlib(') or sreq == 'solvable:prereqmarker':
continue
for dpkg in query.filter(provides=req):
providers.add(pkgmap[dpkg])
if len(providers) == 1 and i not in providers:
deps.update(providers)
providers.clear()

deplist = list(deps)
deps.clear()
depends.append(deplist)
for j in deplist:
rdepends[j].append(i)

return (packages, depends, rdepends)

def kosaraju(self, graph, rgraph):
"""
Run Kosaraju's algorithm to find strongly connected components
in the graph, and return the components without any incoming edges.
"""
N = len(graph)
rstack = []
stack = []
idx = []
tag = [False] * N

# do depth-first searches in the graph
# and push nodes to rstack "on the way up"
# until all nodes have been pushed.
# tag nodes so we don't visit them more than once
for u in range(N):
if tag[u]:
continue

stack.append(u)
idx.append(len(graph[u]))
tag[u] = True
while stack:
u = stack[-1]
i = idx[-1]

if i:
i -= 1
idx[-1] = i
v = graph[u][i]
if not tag[v]:
stack.append(v)
idx.append(len(graph[v]))
tag[v] = True
else:
stack.pop()
idx.pop()
rstack.append(u)

# now searches beginning at nodes popped from
# rstack in the graph with all edges reversed
# will give us the strongly connected components.
# the incoming edges to each component is the
# union of incoming edges to each node in the
# component minus the incoming edges from
# component nodes themselves.
# now all nodes are tagged, so this time let's
# remove the tags as we visit each node.
leaves = []
scc = []
sccredges = set()
while rstack:
v = rstack.pop()
if not tag[v]:
continue

stack.append(v)
tag[v] = False
while stack:
v = stack.pop()
redges = rgraph[v]
scc.append(v)
sccredges.update(redges)
for u in redges:
if tag[u]:
stack.append(u)
tag[u] = False

sccredges.difference_update(scc)
if not sccredges:
leaves.append(scc.copy())
del scc[:]
sccredges.clear()

return leaves

def run(self):
(packages, depends, rdepends) = self.buildgraph()
leaves = self.kosaraju(depends, rdepends)
for scc in leaves:
for i, pkg in enumerate(scc):
real_pkg = packages[pkg]
scc[i] = "%s;%s" % (real_pkg.name, real_pkg.arch)
scc.sort()
leaves.sort(key=lambda scc: scc[0])

print("\n".join(["::".join(scc) for scc in leaves]))

0 comments on commit 2835821

Please sign in to comment.