Sign Up
Log In
Log In
or
Sign Up
Places
All Projects
Status Monitor
Collapse sidebar
home:adrianSuSE
build
pbuild
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File pbuild of Package build
#!/usr/bin/perl ################################################################ # # Copyright (c) 2021 SUSE LLC # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or 3 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program (see the file COPYING); if not, write to the # Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA # ################################################################ BEGIN { if (!$::ENV{'BUILD_DIR'} && $0 ne '-' && $0 ne '-e' && -e $0 && ! -e '/etc/build.conf') { use Cwd (); my $p = Cwd::abs_path($0); $::ENV{'BUILD_DIR'} = $p if $p =~ s/\/[^\/]+$// && $p ne '/usr/lib/build' && -d "$p/PBuild"; } unshift @INC, ($::ENV{'BUILD_DIR'} && ! -e '/etc/build.conf' ? $::ENV{'BUILD_DIR'} : '/usr/lib/build'); } use strict; use Data::Dumper; use POSIX; use Cwd (); use Build; use PBuild::Source; use PBuild::Recipe; use PBuild::AssetMgr; use PBuild::RepoMgr; use PBuild::LocalRepo; use PBuild::RemoteRepo; use PBuild::Multibuild; use PBuild::Link; use PBuild::Checker; use PBuild::Options; use PBuild::Result; use Build::Download; use PBuild::Preset; use PBuild::Distro; use PBuild::Repoquery; my $libbuild = $INC[0]; # parse options my ($opts, @dirs) = PBuild::Options::parse_options(@ARGV); PBuild::Options::usage(0) if $opts->{'help'}; die("Usage: pbuild [options] [dir]\n") if @dirs > 1; my $dir = @dirs ? $dirs[0] : '.'; $dir = Cwd::abs_path($dir) if $dir !~ /^\//; $dir =~ s/(.)\/+$/$1/s; # autodetect single mode if (!exists($opts->{'single'}) && PBuild::Recipe::looks_like_packagedir($dir)) { $opts->{'single'} = $1 if $dir =~ s/\/([^\/]+)$//; } if ($opts->{'list-presets'}) { PBuild::Preset::list_presets($dir); exit; } # read preset my $preset = PBuild::Preset::read_presets($dir, $opts->{'preset'}); my $hostarch = $opts->{'hostarch'}; if (!$hostarch) { $hostarch = (POSIX::uname())[4]; die("cannot determine hostarch\n") unless $hostarch; $hostarch = 'armv6hl' if $hostarch eq 'armv6l'; $hostarch = 'armv7hl' if $hostarch eq 'armv7l'; } # read old options my $oldlastdata = PBuild::Util::retrieve("$dir/.pbuild/lastdata", 1) || {}; my $oldreponame = ($preset || {})->{'name'}; my $oldmyarch = ($preset || {})->{'arch'} || $hostarch; my $olddist = ($preset || {})->{'config'} || $opts->{'dist'}; if (!$oldreponame && $olddist) { $oldreponame = $olddist->[0]; $oldreponame = $1 if $oldreponame =~ /^obs:\/.*?([^\/]+)\/standard\/*$/s; $oldreponame =~ s/.*\///; $oldreponame =~ s/\.conf$//; $oldreponame =~ s/[:\s]+/_/g; } my $oldbuilddir = $oldlastdata->{'builddir'}; if ($oldreponame || $olddist) { $oldbuilddir = $oldreponame && $oldreponame ne $oldmyarch ? "$dir/_build.$oldreponame.$oldmyarch" : "$dir/_build.$oldmyarch"; } my $oldlastopts = PBuild::Util::retrieve("$oldbuilddir/.pbuild/lastopts", 1) || {}; my $newlastopts = PBuild::Options::merge_old_options($opts, $oldlastopts); # tweak options die("Option --shell only works with --single\n") if $opts->{'shell'} && !$opts->{'single'}; die("Option --shell-after-fail only works with --single\n") if $opts->{'shell-after-fail'} && !$opts->{'single'}; $opts->{'showlog'} = 1 if $opts->{'shell'} || $opts->{'shell-after-fail'} || $opts->{'single'}; $opts->{'buildjobs'} = 1 if $opts->{'showlog'} || $opts->{'single'}; # set defaults $opts->{'libbuild'} = $libbuild; if ($<) { $opts->{'vm-type'} ||= 'kvm'; if (!$opts->{'root'}) { my $username = $ENV{LOGNAME} || $ENV{USER} || getpwuid($<) || sprintf("%u", $<); $opts->{'root'} = "/var/tmp/build-root-$username"; } } $opts->{'root'} ||= '/var/tmp/build-root'; $opts->{'root'} = Cwd::abs_path($opts->{'root'}) if $opts->{'root'} !~ /^\//; $opts->{'root'} =~ s/(.)\/+$/$1/s; $opts->{'configdir'} ||= "$libbuild/configs"; $opts->{'hostarch'} = $hostarch; $opts->{'buildjobs'} = 1 unless $opts->{'buildjobs'}; $opts->{'buildjobs'} = 32 if $opts->{'buildjobs'} > 32; # apply presets PBuild::Preset::apply_preset($opts, $preset) if $preset; print "Using default preset: $preset->{'name'}\n" if $preset && !$opts->{'preset'}; my $reponame = $opts->{'reponame'}; if (!$reponame && $opts->{'dist'}) { $reponame = $opts->{'dist'}->[0]; $reponame = $1 if $reponame =~ /^obs:\/.*?([^\/]+)\/standard\/*$/s; $reponame =~ s/.*\///; $reponame =~ s/\.conf$//; $reponame =~ s/[:\s]+/_/g; } $opts->{'arch'} ||= $hostarch; my $myarch = $opts->{'arch'}; my $builddir = $reponame && $reponame ne $myarch ? "$dir/_build.$reponame.$myarch" : "$dir/_build.$myarch"; # update lastdata and lastopts my $newlastdata = { 'builddir' => $builddir }; eval { PBuild::Util::mkdir_p("$dir/.pbuild") ; PBuild::Util::store_unless_identical("$dir/.pbuild/.lastdata.$$", "$dir/.pbuild/lastdata", $newlastdata, $oldlastdata) }; eval { PBuild::Util::mkdir_p("$builddir/.pbuild") ; PBuild::Util::store_unless_identical("$builddir/.pbuild/.lastopts.$$", "$builddir/.pbuild/lastopts", $newlastopts, $oldlastopts) }; if ($opts->{'result-code'} || $opts->{'result-pkg'}) { PBuild::Result::print_result($opts, $builddir); exit; } my $cross = $myarch ne $hostarch && $opts->{'hostrepo'} ? 1 : 0; my @baseconfigs; my $distcnt = 0; my @baseobsrepos; for my $dist (@{$opts->{'dist'} || []}) { $distcnt++; if ($dist =~ /^zypp:/) { $dist = PBuild::Distro::guess_distro($myarch); push @{$opts->{'repo'}}, 'zypp:/' unless @{$opts->{'repo'} || []}; } if ($dist =~ /^https?:\/\//) { my ($config) = Build::Download::fetch($dist); push @baseconfigs, $config; } elsif ($dist =~ /^obs:\//) { my $islast = $distcnt == @{$opts->{'dist'} || []} ? 1 : 0; my ($obsconfigs, $obsrepos) = PBuild::OBS::fetch_all_configs($dist, $opts, $islast); push @baseconfigs, @$obsconfigs; push @baseobsrepos, @$obsrepos; } elsif ($dist =~ /^empty:/) { next; } elsif (-e "$dir/_configs/$opts->{'dist'}.conf") { my $c = Build::slurp_config_file("$dir/_configs/$dist.conf"); push @baseconfigs, join("\n", @$c); } else { my $baseconfigfile = Build::find_config_file($dist, $opts->{'configdir'}); my $c = Build::slurp_config_file($baseconfigfile); push @baseconfigs, join("\n", @$c); } } # set repos from obs, does not work well with "mixed" configs push @{$opts->{'repo'}}, @baseobsrepos if @baseobsrepos && !$opts->{'repo'}; my $localconfig = -s "$dir/_config" ? PBuild::Util::readstr("$dir/_config") : ''; $localconfig = "\n%define _repository $reponame\n\n$localconfig" if $reponame && $localconfig ne ''; my $buildconfig = Build::combine_configs(reverse(@baseconfigs), $localconfig); my $bconf = Build::read_config($myarch, [ split("\n", $buildconfig) ]); my $bconf_host = $cross ? Build::read_config($hostarch, [ split("\n", $buildconfig) ]) : undef; # make sure our config includes some basic setup if (!@{($bconf_host || $bconf)->{'preinstall'} || []}) { my @presetnames = PBuild::Preset::known_presets($dir); if (@presetnames) { print("Please specify a distribution or a preset!\n\n"); PBuild::Preset::list_presets($dir); exit; } else { print("Please specify a distribution!\n\n"); } PBuild::Options::usage(1); } # default to repo/registry from config if not set push @{$opts->{'repo'}}, 'config:' unless @{$opts->{'repo'} || []}; push @{$opts->{'registry'}}, 'config:' unless @{$opts->{'registry'} || []}; push @{$opts->{'assets'}}, 'config:' unless @{$opts->{'assets'} || []}; push @{$opts->{'hostrepo'}}, 'config:' if $cross && !@{$opts->{'hostrepo'} || []}; # substitute config: with values from config for (splice(@{$opts->{'repo'}})) { push @{$opts->{'repo'}}, $_; splice(@{$opts->{'repo'}}, -1, 1, reverse(@{$bconf->{'repourl'}})) if $_ eq 'config:'; } for (splice(@{$opts->{'registry'}})) { push @{$opts->{'registry'}}, $_; splice(@{$opts->{'registry'}}, -1, 1, reverse(@{$bconf->{'registryurl'}})) if $_ eq 'config:'; } for (splice(@{$opts->{'assets'}})) { push @{$opts->{'assets'}}, $_; splice(@{$opts->{'assets'}}, -1, 1, reverse(@{$bconf->{'assetsurl'}})) if $_ eq 'config:'; } if ($cross) { for (splice(@{$opts->{'hostrepo'}})) { push @{$opts->{'hostrepo'}}, $_; splice(@{$opts->{'hostrepo'}}, -1, 1, reverse(@{$bconf_host->{'repourl'}})) if $_ eq 'config:'; } } # expand the zypp:// repo PBuild::RemoteRepo::expand_zypp_repo($opts->{'repo'}); PBuild::RemoteRepo::expand_zypp_repo($opts->{'hostrepo'}) if $cross; print "starting project builder\n"; print " source directory: $dir\n"; print " result directory: $builddir\n"; print " single build: $opts->{'single'}\n" if $opts->{'single'}; print " build area: $opts->{'root'}\n"; print " architecture: $myarch\n"; print " host architecture: $hostarch\n" if $cross; print " preset: $preset->{'name'}\n" if $preset; if (@{$opts->{'dist'} || []}) { print " build config:\n"; print " - $_\n" for @{$opts->{'dist'}}; } if (@{$opts->{'repo'} || []}) { print " repositories:\n"; print " - $_\n" for @{$opts->{'repo'}}; } if (@{$opts->{'assets'} || []}) { print " assets:\n"; print " - $_\n" for @{$opts->{'assets'}}; } if ($cross && @{$opts->{'hostrepo'} || []}) { print " host repositories:\n"; print " - $_\n" for @{$opts->{'hostrepo'}}; } print "searching for packages\n"; my @pkgs = PBuild::Source::find_packages($dir); die("no packages found in '$dir'\n") unless @pkgs; print "found ".PBuild::Util::plural(scalar(@pkgs), 'package')."\n"; my $assetmgr = PBuild::AssetMgr::create("$dir/.pbuild/_assets"); for my $assetsurl (@{$opts->{'assets'} || []}) { $assetmgr->add_assetshandler($assetsurl); } print "getting package information\n"; my %pkgsrc; for my $pkg (@pkgs) { my ($files, $source_assets) = PBuild::Source::list_package("$dir/$pkg"); my $p = { 'pkg' => $pkg, 'dir' => "$dir/$pkg", 'files' => $files, 'srcmd5' => PBuild::Source::calc_srcmd5($files), 'srcmd5' => PBuild::Source::calc_srcmd5($files), }; $p->{'source_assets'} = $source_assets if @{$source_assets || []}; $pkgsrc{$pkg} = $p; } # handle local links and multibuild packages my $nlink = PBuild::Link::count_links(\%pkgsrc); if ($nlink) { print "expanding ".PBuild::Util::plural($nlink, 'package link')."\n"; PBuild::Link::expand_links(\%pkgsrc); } my $nmultibuild = PBuild::Multibuild::count_multibuilds(\%pkgsrc); if ($nmultibuild) { print "expanding ".PBuild::Util::plural($nmultibuild, 'multibuild package')."\n"; PBuild::Multibuild::expand_multibuilds(\%pkgsrc); } # make sure that we know the package if --single is used if ($opts->{'single'}) { $opts->{'single'} .= ":$opts->{'single-flavor'}" if $opts->{'single-flavor'}; die("--single: unknown package $opts->{'single'}\n") if !$pkgsrc{$opts->{'single'}}; } @pkgs = sort keys %pkgsrc; # handle onlybuild/excludebuild from the build config if (exists $bconf->{'buildflags:excludebuild'}) { my %excludebuild; /^excludebuild:(.*)$/s && ($excludebuild{$1} = 1) for @{$bconf->{'buildflags'} || []}; if (%excludebuild) { for my $pkg (@pkgs) { my $p = $pkgsrc{$pkg}; my $releasename = $p->{'releasename'} || $pkg; $p->{'error'} = "excluded:project config excludebuild list" if $excludebuild{$pkg} || $excludebuild{$releasename}; } } } if (exists $bconf->{'buildflags:onlybuild'}) { my %onlybuild; /^onlybuild:(.*)$/s && ($onlybuild{$1} = 1) for @{$bconf->{'buildflags'} || []}; if (%onlybuild) { for my $pkg (@pkgs) { my $p = $pkgsrc{$pkg}; my $releasename = $p->{'releasename'} || $pkg; $p->{'error'} = "excluded:project config onlybuild list" unless $onlybuild{$pkg} || $onlybuild{$releasename}; } } } # parse all recipes in the packages to get dependency information print "parsing ".PBuild::Util::plural(scalar(@pkgs), 'recipe file')."\n"; my %containertags; my $buildtype = $bconf->{'type'} || ''; $buildtype = 'spec' if !$buildtype || $buildtype eq 'UNDEFINED'; for my $pkg (@pkgs) { my $p = $pkgsrc{$pkg}; PBuild::Recipe::parse($bconf, $p, $buildtype, $myarch, $bconf_host, $hostarch); next if $opts->{'single'} && $pkg ne $opts->{'single'}; if ($p->{'buildtype'} && ($p->{'buildtype'} eq 'kiwi' || $p->{'buildtype'} eq 'docker') && !$p->{'error'}) { $containertags{substr($_, 10)} = 1 for grep {/^container:/} @{$p->{'dep'} || []}; } } my @containertags = sort keys %containertags; # split into target/native packages my @pkgs_target = @pkgs; my @pkgs_native; if ($cross) { @pkgs_target = grep {!$pkgsrc{$_}->{'native'}} @pkgs; @pkgs_native = grep {$pkgsrc{$_}->{'native'}} @pkgs; } # search for assets for my $pkg (@pkgs) { next if $opts->{'single'} && $pkg ne $opts->{'single'}; $assetmgr->find_assets($pkgsrc{$pkg}); } #FIXME for my $pkg (@pkgs) { my $p = $pkgsrc{$pkg}; $p->{'useforbuildenabled'} = 1; } # force rebuilds if requested if ($opts->{'rebuild-code'} || $opts->{'rebuild-pkg'}) { my %codefilter = map {$_ => 1} @{$opts->{'rebuild-code'} || []}; my %pkgfilter = map {$_ => 1} @{$opts->{'rebuild-pkg'} || []}; for my $pkg (sort keys %pkgfilter) { die("rebuild: unknown package $pkg\n") unless $pkgsrc{$pkg}; } my $oldresult = {}; $oldresult = PBuild::Util::retrieve("$builddir/.pbuild/_result") if %codefilter && !$codefilter{'all'}; for my $pkg (@pkgs) { my $p = $pkgsrc{$pkg}; my $code = ($oldresult->{$pkg} || {})->{'code'} || 'unknown'; next if %pkgfilter && !$pkgfilter{$pkg}; next if %codefilter && !$codefilter{'all'} && !$codefilter{$code}; $p->{'force_rebuild'} = 1; } } # delete obsolete entries from builddir PBuild::LocalRepo::cleanup_builddir($builddir, \%pkgsrc) unless $opts->{'single'}; # setup the repositories and registries my $repomgr = PBuild::RepoMgr::create(); my @repos; my @hostrepos; print "fetching metadata of the local ".(@pkgs_native ? 'repos' : 'repo')."\n"; push @repos, $repomgr->addlocalrepo($bconf, $myarch, $builddir, \%pkgsrc, \@pkgs_target); push @hostrepos, $repomgr->addlocalrepo($bconf_host, $hostarch, $builddir, \%pkgsrc, \@pkgs_native) if @pkgs_native; print "fetching metadata of ".PBuild::Util::plural(scalar(@{$opts->{'repo'}}) + ($cross ? scalar(@{$opts->{'hostrepo'}}) : 0), 'remote repo')."\n"; for my $repourl (@{$opts->{'repo'}}) { if ($repourl =~ /^registry@(.+)/) { push @repos, $repomgr->addremoteregistry($bconf, $myarch, $builddir, $1, \@containertags); } else { push @repos, $repomgr->addremoterepo($bconf, $myarch, $builddir, $repourl, $buildtype, $opts); } } if ($cross) { for my $repourl (@{$opts->{'hostrepo'}}) { push @hostrepos, $repomgr->addremoterepo($bconf_host, $hostarch, $builddir, $repourl, $buildtype, $opts); } } if (@{$opts->{'registry'} || []} && @containertags) { print "fetching remote registry metadata\n"; for my $registry (@{$opts->{'registry'} || []}) { push @repos, $repomgr->addremoteregistry($bconf, $myarch, $builddir, $registry, \@containertags); } } if ($opts->{'repoquery'}) { PBuild::Repoquery::repoquery($bconf, $myarch, \@repos, $opts->{'repoquery'}, $opts); exit; } if ($opts->{'repoquery-host'}) { die("No cross building configured\n") unless $cross; PBuild::Repoquery::repoquery($bconf_host, $hostarch, \@hostrepos, $opts->{'repoquery-host'}, $opts); exit; } # load lastcheck cache my %lastcheck; if (-s "$builddir/.pbuild/_lastcheck") { my $oldlastcheck = PBuild::Util::retrieve("$builddir/.pbuild/_lastcheck", 1) || {}; for my $pkg (@pkgs) { my $old = $oldlastcheck->{$pkg}; $lastcheck{$pkg} = $old if $old && length($old) > 96; } } # tweak package list if we're just looking at one package if ($opts->{'single'}) { my $pkg = $opts->{'single'}; @pkgs = ( $pkg ); $pkgsrc{$pkg}->{'force_rebuild'} = 1; } # split deps if cross building if ($cross) { for my $pkg (@pkgs) { my $p = $pkgsrc{$pkg}; PBuild::Recipe::split_hostdeps($p, $bconf); } } # setup builders my @builders; for my $no (1..$opts->{'buildjobs'}) { my $broot = $opts->{'root'}; if ($opts->{'buildjobs'} > 1) { $broot .= '/%I' if $broot !~ /%I/; $broot =~ s/%I/$no/g; } push @builders, { 'name' => $no, 'root' => $broot, 'idx' => scalar(@builders), 'nbuilders' => $opts->{'buildjobs'}, }; } my $ctx; my $runs = 0; # the big loop: while there is something to do while (1) { # create and setup checker if (!$ctx) { $ctx = PBuild::Checker::create($bconf, $myarch, $buildtype, \%pkgsrc, $builddir, $opts, $repomgr, $assetmgr); $ctx->{'hostarch'} = $hostarch; $ctx->{'bconf_host'} = $bconf_host if $cross; print "preparing package pool\n" unless $runs; $ctx->prepare(\@repos, \@hostrepos); print "expanding dependencies\n" unless $runs; $ctx->pkgexpand(@pkgs); if (@pkgs > 1) { print "sorting packages\n" unless $runs; if (@pkgs_native) { @pkgs_native = $ctx->pkgsort(@pkgs_native); @pkgs_target = $ctx->pkgsort(@pkgs_target); @pkgs = (@pkgs_native, @pkgs_target); } else { @pkgs = $ctx->pkgsort(@pkgs); } } } $runs++; $ctx->{'buildconfig'} = $buildconfig; $ctx->{'lastcheck'} = \%lastcheck; # check status of all packages my $result = $ctx->pkgcheck(\@builders, @pkgs); # mix in old result from other packages if in single package mode if ($opts->{'single'}) { my $pkg = $opts->{'single'}; my $oldresult = PBuild::Util::retrieve("$builddir/.pbuild/_result", 1) || {}; $oldresult->{$pkg} = $result->{$pkg}; $result = $oldresult; my $code = $result->{$pkg}->{'code'}; if ($code ne 'failed' && $code ne 'succeeded' && $code ne 'building') { $code .= ": $result->{$pkg}->{'details'}" if $result->{$pkg}->{'details'}; print "$pkg: $code\n"; } } # update on-disk data PBuild::Util::mkdir_p("$builddir/.pbuild"); PBuild::Util::store("$builddir/.pbuild/._result.$$", "$builddir/.pbuild/_result", $result); PBuild::Util::store("$builddir/.pbuild/._lastcheck.$$", "$builddir/.pbuild/_lastcheck", \%lastcheck); # get list of building jobs my @building = map {$_->{'job'}} grep {$_->{'job'}} @builders; last unless @building; # wait for one job to finish my $job = PBuild::Job::waitjob($opts, @building); for (@builders) { delete $_->{'job'} if $_->{'job'} && $_->{'job'} == $job; } # process finished job my ($code, $buildresult) = PBuild::Job::finishjob($job); my $p = $job->{'pdata'}; delete $p->{'force_rebuild'}; my $duration = $job->{'endtime'} - $job->{'starttime'}; $duration = sprintf("%d:%02d", int($duration / 60), $duration % 60); my $bid = ($job->{'nbuilders'} || 1) > 1 ? "$job->{'name'}: " : ''; print "${bid}finished $p->{'pkg'}/$p->{'recipe'} after ${duration}: $code\n"; my $jobhist = PBuild::BuildResult::makejobhist($p, $code, $job->{'readytime'}, $job->{'starttime'}, $job->{'endtime'}, $job->{'reason'}, $job->{'hostarch'}); PBuild::BuildResult::addjobhist($builddir, $jobhist); # integrate build artifacts and extra files my $bininfo = PBuild::BuildResult::integrate_job($builddir, $job, $code, $buildresult); # if the build was successful, update artifact information and the local repo if ($bininfo) { PBuild::LocalRepo::update_gbininfo($builddir, $p->{'pkg'}, $bininfo); if ($p->{'useforbuildenabled'}) { # update with new local bin information if ($p->{'native'}) { $repomgr->updatelocalrepo($bconf, $hostarch, $builddir, \%pkgsrc, \@pkgs_native); } else { $repomgr->updatelocalrepo($bconf, $myarch, $builddir, \%pkgsrc, \@pkgs_target); } # we also need a new checker undef $ctx; } } } exit PBuild::Result::has_failed($opts, $builddir, $opts->{'single'}) ? 1 : 0 if $opts->{'single'}; # say goodbye print "\npbuild is done:\n"; exit PBuild::Result::print_result($opts, $builddir);
Locations
Projects
Search
Status Monitor
Help
OpenBuildService.org
Documentation
API Documentation
Code of Conduct
Contact
Support
@OBShq
The Open Build Service is an
openSUSE project
.
Sign Up
Log In
Places
Places
All Projects
Status Monitor