File pbuild of Package build
#!/usr/bin/perl
################################################################
#
# Copyright (c) 2021 SUSE LLC
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 or 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program (see the file COPYING); if not, write to the
# Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#
################################################################
BEGIN {
if (!$::ENV{'BUILD_DIR'} && $0 ne '-' && $0 ne '-e' && -e $0 && ! -e '/etc/build.conf') {
use Cwd ();
my $p = Cwd::abs_path($0);
$::ENV{'BUILD_DIR'} = $p if $p =~ s/\/[^\/]+$// && $p ne '/usr/lib/build' && -d "$p/PBuild";
}
unshift @INC, ($::ENV{'BUILD_DIR'} && ! -e '/etc/build.conf' ? $::ENV{'BUILD_DIR'} : '/usr/lib/build');
}
use strict;
use Data::Dumper;
use POSIX;
use Cwd ();
use Build;
use PBuild::Source;
use PBuild::Recipe;
use PBuild::AssetMgr;
use PBuild::RepoMgr;
use PBuild::LocalRepo;
use PBuild::RemoteRepo;
use PBuild::Multibuild;
use PBuild::Link;
use PBuild::Checker;
use PBuild::Options;
use PBuild::Result;
use Build::Download;
use PBuild::Preset;
use PBuild::Distro;
use PBuild::Repoquery;
my $libbuild = $INC[0];
# parse options
my ($opts, @dirs) = PBuild::Options::parse_options(@ARGV);
PBuild::Options::usage(0) if $opts->{'help'};
die("Usage: pbuild [options] [dir]\n") if @dirs > 1;
my $dir = @dirs ? $dirs[0] : '.';
$dir = Cwd::abs_path($dir) if $dir !~ /^\//;
$dir =~ s/(.)\/+$/$1/s;
# autodetect single mode
if (!exists($opts->{'single'}) && PBuild::Recipe::looks_like_packagedir($dir)) {
$opts->{'single'} = $1 if $dir =~ s/\/([^\/]+)$//;
}
if ($opts->{'list-presets'}) {
PBuild::Preset::list_presets($dir);
exit;
}
# read preset
my $preset = PBuild::Preset::read_presets($dir, $opts->{'preset'});
my $hostarch = $opts->{'hostarch'};
if (!$hostarch) {
$hostarch = (POSIX::uname())[4];
die("cannot determine hostarch\n") unless $hostarch;
$hostarch = 'armv6hl' if $hostarch eq 'armv6l';
$hostarch = 'armv7hl' if $hostarch eq 'armv7l';
}
# read old options
my $oldlastdata = PBuild::Util::retrieve("$dir/.pbuild/lastdata", 1) || {};
my $oldreponame = ($preset || {})->{'name'};
my $oldmyarch = ($preset || {})->{'arch'} || $hostarch;
my $olddist = ($preset || {})->{'config'} || $opts->{'dist'};
if (!$oldreponame && $olddist) {
$oldreponame = $olddist->[0];
$oldreponame = $1 if $oldreponame =~ /^obs:\/.*?([^\/]+)\/standard\/*$/s;
$oldreponame =~ s/.*\///;
$oldreponame =~ s/\.conf$//;
$oldreponame =~ s/[:\s]+/_/g;
}
my $oldbuilddir = $oldlastdata->{'builddir'};
if ($oldreponame || $olddist) {
$oldbuilddir = $oldreponame && $oldreponame ne $oldmyarch ? "$dir/_build.$oldreponame.$oldmyarch" : "$dir/_build.$oldmyarch";
}
my $oldlastopts = PBuild::Util::retrieve("$oldbuilddir/.pbuild/lastopts", 1) || {};
my $newlastopts = PBuild::Options::merge_old_options($opts, $oldlastopts);
# tweak options
die("Option --shell only works with --single\n") if $opts->{'shell'} && !$opts->{'single'};
die("Option --shell-after-fail only works with --single\n") if $opts->{'shell-after-fail'} && !$opts->{'single'};
$opts->{'showlog'} = 1 if $opts->{'shell'} || $opts->{'shell-after-fail'} || $opts->{'single'};
$opts->{'buildjobs'} = 1 if $opts->{'showlog'} || $opts->{'single'};
# set defaults
$opts->{'libbuild'} = $libbuild;
if ($<) {
$opts->{'vm-type'} ||= 'kvm';
if (!$opts->{'root'}) {
my $username = $ENV{LOGNAME} || $ENV{USER} || getpwuid($<) || sprintf("%u", $<);
$opts->{'root'} = "/var/tmp/build-root-$username";
}
}
$opts->{'root'} ||= '/var/tmp/build-root';
$opts->{'root'} = Cwd::abs_path($opts->{'root'}) if $opts->{'root'} !~ /^\//;
$opts->{'root'} =~ s/(.)\/+$/$1/s;
$opts->{'configdir'} ||= "$libbuild/configs";
$opts->{'hostarch'} = $hostarch;
$opts->{'buildjobs'} = 1 unless $opts->{'buildjobs'};
$opts->{'buildjobs'} = 32 if $opts->{'buildjobs'} > 32;
# apply presets
PBuild::Preset::apply_preset($opts, $preset) if $preset;
print "Using default preset: $preset->{'name'}\n" if $preset && !$opts->{'preset'};
my $reponame = $opts->{'reponame'};
if (!$reponame && $opts->{'dist'}) {
$reponame = $opts->{'dist'}->[0];
$reponame = $1 if $reponame =~ /^obs:\/.*?([^\/]+)\/standard\/*$/s;
$reponame =~ s/.*\///;
$reponame =~ s/\.conf$//;
$reponame =~ s/[:\s]+/_/g;
}
$opts->{'arch'} ||= $hostarch;
my $myarch = $opts->{'arch'};
my $builddir = $reponame && $reponame ne $myarch ? "$dir/_build.$reponame.$myarch" : "$dir/_build.$myarch";
# update lastdata and lastopts
my $newlastdata = { 'builddir' => $builddir };
eval { PBuild::Util::mkdir_p("$dir/.pbuild") ; PBuild::Util::store_unless_identical("$dir/.pbuild/.lastdata.$$", "$dir/.pbuild/lastdata", $newlastdata, $oldlastdata) };
eval { PBuild::Util::mkdir_p("$builddir/.pbuild") ; PBuild::Util::store_unless_identical("$builddir/.pbuild/.lastopts.$$", "$builddir/.pbuild/lastopts", $newlastopts, $oldlastopts) };
if ($opts->{'result-code'} || $opts->{'result-pkg'}) {
PBuild::Result::print_result($opts, $builddir);
exit;
}
my $cross = $myarch ne $hostarch && $opts->{'hostrepo'} ? 1 : 0;
my @baseconfigs;
my $distcnt = 0;
my @baseobsrepos;
for my $dist (@{$opts->{'dist'} || []}) {
$distcnt++;
if ($dist =~ /^zypp:/) {
$dist = PBuild::Distro::guess_distro($myarch);
push @{$opts->{'repo'}}, 'zypp:/' unless @{$opts->{'repo'} || []};
}
if ($dist =~ /^https?:\/\//) {
my ($config) = Build::Download::fetch($dist);
push @baseconfigs, $config;
} elsif ($dist =~ /^obs:\//) {
my $islast = $distcnt == @{$opts->{'dist'} || []} ? 1 : 0;
my ($obsconfigs, $obsrepos) = PBuild::OBS::fetch_all_configs($dist, $opts, $islast);
push @baseconfigs, @$obsconfigs;
push @baseobsrepos, @$obsrepos;
} elsif ($dist =~ /^empty:/) {
next;
} elsif (-e "$dir/_configs/$opts->{'dist'}.conf") {
my $c = Build::slurp_config_file("$dir/_configs/$dist.conf");
push @baseconfigs, join("\n", @$c);
} else {
my $baseconfigfile = Build::find_config_file($dist, $opts->{'configdir'});
my $c = Build::slurp_config_file($baseconfigfile);
push @baseconfigs, join("\n", @$c);
}
}
# set repos from obs, does not work well with "mixed" configs
push @{$opts->{'repo'}}, @baseobsrepos if @baseobsrepos && !$opts->{'repo'};
my $localconfig = -s "$dir/_config" ? PBuild::Util::readstr("$dir/_config") : '';
$localconfig = "\n%define _repository $reponame\n\n$localconfig" if $reponame && $localconfig ne '';
my $buildconfig = Build::combine_configs(reverse(@baseconfigs), $localconfig);
my $bconf = Build::read_config($myarch, [ split("\n", $buildconfig) ]);
my $bconf_host = $cross ? Build::read_config($hostarch, [ split("\n", $buildconfig) ]) : undef;
# make sure our config includes some basic setup
if (!@{($bconf_host || $bconf)->{'preinstall'} || []}) {
my @presetnames = PBuild::Preset::known_presets($dir);
if (@presetnames) {
print("Please specify a distribution or a preset!\n\n");
PBuild::Preset::list_presets($dir);
exit;
} else {
print("Please specify a distribution!\n\n");
}
PBuild::Options::usage(1);
}
# default to repo/registry from config if not set
push @{$opts->{'repo'}}, 'config:' unless @{$opts->{'repo'} || []};
push @{$opts->{'registry'}}, 'config:' unless @{$opts->{'registry'} || []};
push @{$opts->{'assets'}}, 'config:' unless @{$opts->{'assets'} || []};
push @{$opts->{'hostrepo'}}, 'config:' if $cross && !@{$opts->{'hostrepo'} || []};
# substitute config: with values from config
for (splice(@{$opts->{'repo'}})) {
push @{$opts->{'repo'}}, $_;
splice(@{$opts->{'repo'}}, -1, 1, reverse(@{$bconf->{'repourl'}})) if $_ eq 'config:';
}
for (splice(@{$opts->{'registry'}})) {
push @{$opts->{'registry'}}, $_;
splice(@{$opts->{'registry'}}, -1, 1, reverse(@{$bconf->{'registryurl'}})) if $_ eq 'config:';
}
for (splice(@{$opts->{'assets'}})) {
push @{$opts->{'assets'}}, $_;
splice(@{$opts->{'assets'}}, -1, 1, reverse(@{$bconf->{'assetsurl'}})) if $_ eq 'config:';
}
if ($cross) {
for (splice(@{$opts->{'hostrepo'}})) {
push @{$opts->{'hostrepo'}}, $_;
splice(@{$opts->{'hostrepo'}}, -1, 1, reverse(@{$bconf_host->{'repourl'}})) if $_ eq 'config:';
}
}
# expand the zypp:// repo
PBuild::RemoteRepo::expand_zypp_repo($opts->{'repo'});
PBuild::RemoteRepo::expand_zypp_repo($opts->{'hostrepo'}) if $cross;
print "starting project builder\n";
print " source directory: $dir\n";
print " result directory: $builddir\n";
print " single build: $opts->{'single'}\n" if $opts->{'single'};
print " build area: $opts->{'root'}\n";
print " architecture: $myarch\n";
print " host architecture: $hostarch\n" if $cross;
print " preset: $preset->{'name'}\n" if $preset;
if (@{$opts->{'dist'} || []}) {
print " build config:\n";
print " - $_\n" for @{$opts->{'dist'}};
}
if (@{$opts->{'repo'} || []}) {
print " repositories:\n";
print " - $_\n" for @{$opts->{'repo'}};
}
if (@{$opts->{'assets'} || []}) {
print " assets:\n";
print " - $_\n" for @{$opts->{'assets'}};
}
if ($cross && @{$opts->{'hostrepo'} || []}) {
print " host repositories:\n";
print " - $_\n" for @{$opts->{'hostrepo'}};
}
print "searching for packages\n";
my @pkgs = PBuild::Source::find_packages($dir);
die("no packages found in '$dir'\n") unless @pkgs;
print "found ".PBuild::Util::plural(scalar(@pkgs), 'package')."\n";
my $assetmgr = PBuild::AssetMgr::create("$dir/.pbuild/_assets");
for my $assetsurl (@{$opts->{'assets'} || []}) {
$assetmgr->add_assetshandler($assetsurl);
}
print "getting package information\n";
my %pkgsrc;
for my $pkg (@pkgs) {
my ($files, $source_assets) = PBuild::Source::list_package("$dir/$pkg");
my $p = {
'pkg' => $pkg,
'dir' => "$dir/$pkg",
'files' => $files,
'srcmd5' => PBuild::Source::calc_srcmd5($files),
'srcmd5' => PBuild::Source::calc_srcmd5($files),
};
$p->{'source_assets'} = $source_assets if @{$source_assets || []};
$pkgsrc{$pkg} = $p;
}
# handle local links and multibuild packages
my $nlink = PBuild::Link::count_links(\%pkgsrc);
if ($nlink) {
print "expanding ".PBuild::Util::plural($nlink, 'package link')."\n";
PBuild::Link::expand_links(\%pkgsrc);
}
my $nmultibuild = PBuild::Multibuild::count_multibuilds(\%pkgsrc);
if ($nmultibuild) {
print "expanding ".PBuild::Util::plural($nmultibuild, 'multibuild package')."\n";
PBuild::Multibuild::expand_multibuilds(\%pkgsrc);
}
# make sure that we know the package if --single is used
if ($opts->{'single'}) {
$opts->{'single'} .= ":$opts->{'single-flavor'}" if $opts->{'single-flavor'};
die("--single: unknown package $opts->{'single'}\n") if !$pkgsrc{$opts->{'single'}};
}
@pkgs = sort keys %pkgsrc;
# handle onlybuild/excludebuild from the build config
if (exists $bconf->{'buildflags:excludebuild'}) {
my %excludebuild;
/^excludebuild:(.*)$/s && ($excludebuild{$1} = 1) for @{$bconf->{'buildflags'} || []};
if (%excludebuild) {
for my $pkg (@pkgs) {
my $p = $pkgsrc{$pkg};
my $releasename = $p->{'releasename'} || $pkg;
$p->{'error'} = "excluded:project config excludebuild list" if $excludebuild{$pkg} || $excludebuild{$releasename};
}
}
}
if (exists $bconf->{'buildflags:onlybuild'}) {
my %onlybuild;
/^onlybuild:(.*)$/s && ($onlybuild{$1} = 1) for @{$bconf->{'buildflags'} || []};
if (%onlybuild) {
for my $pkg (@pkgs) {
my $p = $pkgsrc{$pkg};
my $releasename = $p->{'releasename'} || $pkg;
$p->{'error'} = "excluded:project config onlybuild list" unless $onlybuild{$pkg} || $onlybuild{$releasename};
}
}
}
# parse all recipes in the packages to get dependency information
print "parsing ".PBuild::Util::plural(scalar(@pkgs), 'recipe file')."\n";
my %containertags;
my $buildtype = $bconf->{'type'} || '';
$buildtype = 'spec' if !$buildtype || $buildtype eq 'UNDEFINED';
for my $pkg (@pkgs) {
my $p = $pkgsrc{$pkg};
PBuild::Recipe::parse($bconf, $p, $buildtype, $myarch, $bconf_host, $hostarch);
next if $opts->{'single'} && $pkg ne $opts->{'single'};
if ($p->{'buildtype'} && ($p->{'buildtype'} eq 'kiwi' || $p->{'buildtype'} eq 'docker') && !$p->{'error'}) {
$containertags{substr($_, 10)} = 1 for grep {/^container:/} @{$p->{'dep'} || []};
}
}
my @containertags = sort keys %containertags;
# split into target/native packages
my @pkgs_target = @pkgs;
my @pkgs_native;
if ($cross) {
@pkgs_target = grep {!$pkgsrc{$_}->{'native'}} @pkgs;
@pkgs_native = grep {$pkgsrc{$_}->{'native'}} @pkgs;
}
# search for assets
for my $pkg (@pkgs) {
next if $opts->{'single'} && $pkg ne $opts->{'single'};
$assetmgr->find_assets($pkgsrc{$pkg});
}
#FIXME
for my $pkg (@pkgs) {
my $p = $pkgsrc{$pkg};
$p->{'useforbuildenabled'} = 1;
}
# force rebuilds if requested
if ($opts->{'rebuild-code'} || $opts->{'rebuild-pkg'}) {
my %codefilter = map {$_ => 1} @{$opts->{'rebuild-code'} || []};
my %pkgfilter = map {$_ => 1} @{$opts->{'rebuild-pkg'} || []};
for my $pkg (sort keys %pkgfilter) {
die("rebuild: unknown package $pkg\n") unless $pkgsrc{$pkg};
}
my $oldresult = {};
$oldresult = PBuild::Util::retrieve("$builddir/.pbuild/_result") if %codefilter && !$codefilter{'all'};
for my $pkg (@pkgs) {
my $p = $pkgsrc{$pkg};
my $code = ($oldresult->{$pkg} || {})->{'code'} || 'unknown';
next if %pkgfilter && !$pkgfilter{$pkg};
next if %codefilter && !$codefilter{'all'} && !$codefilter{$code};
$p->{'force_rebuild'} = 1;
}
}
# delete obsolete entries from builddir
PBuild::LocalRepo::cleanup_builddir($builddir, \%pkgsrc) unless $opts->{'single'};
# setup the repositories and registries
my $repomgr = PBuild::RepoMgr::create();
my @repos;
my @hostrepos;
print "fetching metadata of the local ".(@pkgs_native ? 'repos' : 'repo')."\n";
push @repos, $repomgr->addlocalrepo($bconf, $myarch, $builddir, \%pkgsrc, \@pkgs_target);
push @hostrepos, $repomgr->addlocalrepo($bconf_host, $hostarch, $builddir, \%pkgsrc, \@pkgs_native) if @pkgs_native;
print "fetching metadata of ".PBuild::Util::plural(scalar(@{$opts->{'repo'}}) + ($cross ? scalar(@{$opts->{'hostrepo'}}) : 0), 'remote repo')."\n";
for my $repourl (@{$opts->{'repo'}}) {
if ($repourl =~ /^registry@(.+)/) {
push @repos, $repomgr->addremoteregistry($bconf, $myarch, $builddir, $1, \@containertags);
} else {
push @repos, $repomgr->addremoterepo($bconf, $myarch, $builddir, $repourl, $buildtype, $opts);
}
}
if ($cross) {
for my $repourl (@{$opts->{'hostrepo'}}) {
push @hostrepos, $repomgr->addremoterepo($bconf_host, $hostarch, $builddir, $repourl, $buildtype, $opts);
}
}
if (@{$opts->{'registry'} || []} && @containertags) {
print "fetching remote registry metadata\n";
for my $registry (@{$opts->{'registry'} || []}) {
push @repos, $repomgr->addremoteregistry($bconf, $myarch, $builddir, $registry, \@containertags);
}
}
if ($opts->{'repoquery'}) {
PBuild::Repoquery::repoquery($bconf, $myarch, \@repos, $opts->{'repoquery'}, $opts);
exit;
}
if ($opts->{'repoquery-host'}) {
die("No cross building configured\n") unless $cross;
PBuild::Repoquery::repoquery($bconf_host, $hostarch, \@hostrepos, $opts->{'repoquery-host'}, $opts);
exit;
}
# load lastcheck cache
my %lastcheck;
if (-s "$builddir/.pbuild/_lastcheck") {
my $oldlastcheck = PBuild::Util::retrieve("$builddir/.pbuild/_lastcheck", 1) || {};
for my $pkg (@pkgs) {
my $old = $oldlastcheck->{$pkg};
$lastcheck{$pkg} = $old if $old && length($old) > 96;
}
}
# tweak package list if we're just looking at one package
if ($opts->{'single'}) {
my $pkg = $opts->{'single'};
@pkgs = ( $pkg );
$pkgsrc{$pkg}->{'force_rebuild'} = 1;
}
# split deps if cross building
if ($cross) {
for my $pkg (@pkgs) {
my $p = $pkgsrc{$pkg};
PBuild::Recipe::split_hostdeps($p, $bconf);
}
}
# setup builders
my @builders;
for my $no (1..$opts->{'buildjobs'}) {
my $broot = $opts->{'root'};
if ($opts->{'buildjobs'} > 1) {
$broot .= '/%I' if $broot !~ /%I/;
$broot =~ s/%I/$no/g;
}
push @builders, {
'name' => $no,
'root' => $broot,
'idx' => scalar(@builders),
'nbuilders' => $opts->{'buildjobs'},
};
}
my $ctx;
my $runs = 0;
# the big loop: while there is something to do
while (1) {
# create and setup checker
if (!$ctx) {
$ctx = PBuild::Checker::create($bconf, $myarch, $buildtype, \%pkgsrc, $builddir, $opts, $repomgr, $assetmgr);
$ctx->{'hostarch'} = $hostarch;
$ctx->{'bconf_host'} = $bconf_host if $cross;
print "preparing package pool\n" unless $runs;
$ctx->prepare(\@repos, \@hostrepos);
print "expanding dependencies\n" unless $runs;
$ctx->pkgexpand(@pkgs);
if (@pkgs > 1) {
print "sorting packages\n" unless $runs;
if (@pkgs_native) {
@pkgs_native = $ctx->pkgsort(@pkgs_native);
@pkgs_target = $ctx->pkgsort(@pkgs_target);
@pkgs = (@pkgs_native, @pkgs_target);
} else {
@pkgs = $ctx->pkgsort(@pkgs);
}
}
}
$runs++;
$ctx->{'buildconfig'} = $buildconfig;
$ctx->{'lastcheck'} = \%lastcheck;
# check status of all packages
my $result = $ctx->pkgcheck(\@builders, @pkgs);
# mix in old result from other packages if in single package mode
if ($opts->{'single'}) {
my $pkg = $opts->{'single'};
my $oldresult = PBuild::Util::retrieve("$builddir/.pbuild/_result", 1) || {};
$oldresult->{$pkg} = $result->{$pkg};
$result = $oldresult;
my $code = $result->{$pkg}->{'code'};
if ($code ne 'failed' && $code ne 'succeeded' && $code ne 'building') {
$code .= ": $result->{$pkg}->{'details'}" if $result->{$pkg}->{'details'};
print "$pkg: $code\n";
}
}
# update on-disk data
PBuild::Util::mkdir_p("$builddir/.pbuild");
PBuild::Util::store("$builddir/.pbuild/._result.$$", "$builddir/.pbuild/_result", $result);
PBuild::Util::store("$builddir/.pbuild/._lastcheck.$$", "$builddir/.pbuild/_lastcheck", \%lastcheck);
# get list of building jobs
my @building = map {$_->{'job'}} grep {$_->{'job'}} @builders;
last unless @building;
# wait for one job to finish
my $job = PBuild::Job::waitjob($opts, @building);
for (@builders) {
delete $_->{'job'} if $_->{'job'} && $_->{'job'} == $job;
}
# process finished job
my ($code, $buildresult) = PBuild::Job::finishjob($job);
my $p = $job->{'pdata'};
delete $p->{'force_rebuild'};
my $duration = $job->{'endtime'} - $job->{'starttime'};
$duration = sprintf("%d:%02d", int($duration / 60), $duration % 60);
my $bid = ($job->{'nbuilders'} || 1) > 1 ? "$job->{'name'}: " : '';
print "${bid}finished $p->{'pkg'}/$p->{'recipe'} after ${duration}: $code\n";
my $jobhist = PBuild::BuildResult::makejobhist($p, $code, $job->{'readytime'}, $job->{'starttime'}, $job->{'endtime'}, $job->{'reason'}, $job->{'hostarch'});
PBuild::BuildResult::addjobhist($builddir, $jobhist);
# integrate build artifacts and extra files
my $bininfo = PBuild::BuildResult::integrate_job($builddir, $job, $code, $buildresult);
# if the build was successful, update artifact information and the local repo
if ($bininfo) {
PBuild::LocalRepo::update_gbininfo($builddir, $p->{'pkg'}, $bininfo);
if ($p->{'useforbuildenabled'}) {
# update with new local bin information
if ($p->{'native'}) {
$repomgr->updatelocalrepo($bconf, $hostarch, $builddir, \%pkgsrc, \@pkgs_native);
} else {
$repomgr->updatelocalrepo($bconf, $myarch, $builddir, \%pkgsrc, \@pkgs_target);
}
# we also need a new checker
undef $ctx;
}
}
}
exit PBuild::Result::has_failed($opts, $builddir, $opts->{'single'}) ? 1 : 0 if $opts->{'single'};
# say goodbye
print "\npbuild is done:\n";
exit PBuild::Result::print_result($opts, $builddir);