This is unfortunately a giant change, as all of the functionality that is encompassed into module-sets currently had to migrate over to multiple separate classes, including the new ksb::ModuleSet class and subclasses. This was a long-overdue change, however, and should allow for accurately tracking a source module-set for a given module. On the other hand this migration of logic has made it easier to understand each of the individual pieces where they stand (e.g. there is no longer a separate expandXMLModules and expandModuleSets). In addition we can properly handle ignore-modules with wildcards just as we do with use-modules (they even use the same matching logic) which means that it is safe to integrate this into master (assuming no extra boogs get added, of course). This will also help with fixing some of the extant module-selection bugs (321883, 299415). BUG:321275 CCBUG:321667wilder
Michael Pyne
13 years ago
parent
928293e6fb
commit
5d638acab0
7 changed files with 711 additions and 425 deletions
@ -0,0 +1,182 @@ |
||||
package ksb::ModuleSet; |
||||
|
||||
# Class: ModuleSet |
||||
# |
||||
# This represents a collective grouping of modules that share common options, |
||||
# and share a common repository (in this case, based on the git-repository-base |
||||
# option, but see also the more common ModuleSet::KDEProjects which is used for |
||||
# the special kde-projects repositories). |
||||
# |
||||
# This is parsed from module-set declarations in the rc-file. |
||||
# |
||||
# The major conceit here is several things: |
||||
# |
||||
# 1. A hash of options to set for each module read into this module set. |
||||
# 2. A list of module search declarations to be used to construct modules for |
||||
# this module set (in the case of kde-projects repository). For other |
||||
# repository types we can still consider it a 'search', but with the |
||||
# understanding that it's a 1:1 mapping to the 'found' module (which may not |
||||
# exist for real). |
||||
# 3. A list of module search declarations to *ignore* from this module set, |
||||
# using the same syntax as used to search for them in 2. This is only really |
||||
# useful at this point for kde-projects repository as everything else requires |
||||
# you to manually specify modules one-by-one (module-sets are only useful here |
||||
# for option grouping as per 1.). |
||||
# 4. A name, which must not be empty, although user-specified names cannot be |
||||
# assumed to be unique. |
||||
# 5. A ksb::PhaseList describing what phases of the build a module should |
||||
# participate in by default. |
||||
# |
||||
# See also: git-repository-base, ModuleSet::KDEProjects, use-modules |
||||
|
||||
use strict; |
||||
use warnings; |
||||
use v5.10; |
||||
no if $] >= 5.018, 'warnings', 'experimental::smartmatch'; |
||||
|
||||
our $VERSION = '0.10'; |
||||
|
||||
use ksb::Debug; |
||||
use ksb::Util; |
||||
use ksb::PhaseList; |
||||
use ksb::BuildContext; |
||||
use Storable qw(dclone); |
||||
|
||||
sub new |
||||
{ |
||||
my ($class, $ctx, $name) = @_; |
||||
$name //= ''; |
||||
|
||||
my $options = { |
||||
name => $name, |
||||
options => { }, |
||||
module_search_decls => [ ], |
||||
module_ignore_decls => [ ], |
||||
phase_list => ksb::PhaseList->new($ctx->phases()->phases()), |
||||
}; |
||||
|
||||
return bless $options, $class; |
||||
} |
||||
|
||||
sub name |
||||
{ |
||||
my $self = shift; |
||||
return $self->{name}; |
||||
} |
||||
|
||||
sub setName |
||||
{ |
||||
my ($self, $name) = @_; |
||||
$self->{name} = $name; |
||||
return; |
||||
} |
||||
|
||||
# Returns a deep-copied hashref, not a hash. |
||||
sub options |
||||
{ |
||||
my $self = shift; |
||||
return dclone($self->{options}); |
||||
} |
||||
|
||||
# Completely replaces stored options with the options given in the provided |
||||
# hashref. |
||||
sub setOptions |
||||
{ |
||||
my ($self, $hashref) = @_; |
||||
$self->{options} = $hashref; |
||||
return; |
||||
} |
||||
|
||||
# Just returns a reference to the existing ksb::PhaseList, there's no way to |
||||
# replace this, though you can alter the underlying phases through the |
||||
# ksb::PhaseList object itself. |
||||
sub phases |
||||
{ |
||||
my $self = shift; |
||||
return $self->{phase_list}; |
||||
} |
||||
|
||||
sub modulesToFind |
||||
{ |
||||
my $self = shift; |
||||
return @{$self->{module_search_decls}}; |
||||
} |
||||
|
||||
sub setModulesToFind |
||||
{ |
||||
my ($self, @moduleDecls) = @_; |
||||
$self->{module_search_decls} = [@moduleDecls]; |
||||
return; |
||||
} |
||||
|
||||
sub modulesToIgnore |
||||
{ |
||||
my $self = shift; |
||||
return @{$self->{module_ignore_decls}}; |
||||
} |
||||
|
||||
sub setModulesToIgnore |
||||
{ |
||||
my ($self, @moduleDecls) = @_; |
||||
$self->{module_ignore_decls} = [@moduleDecls]; |
||||
return; |
||||
} |
||||
|
||||
# Should be called for each new ksb::Module created in order to setup common |
||||
# module options. |
||||
sub _initializeNewModule |
||||
{ |
||||
my ($self, $newModule) = @_; |
||||
|
||||
$newModule->setModuleSet($self); |
||||
$newModule->setScmType('git'); |
||||
$newModule->phases->phases($self->phases()->phases()); |
||||
|
||||
# Dump all options into the existing ksb::Module's options. |
||||
$newModule->setOption(%{$self->options()}); |
||||
} |
||||
|
||||
# This function should be called after options are read and build metadata is |
||||
# available in order to convert this module set to a list of ksb::Module. |
||||
# Any modules ignored by this module set are excluded from the returned list. |
||||
# The modules returned have not been added to the build context. |
||||
sub convertToModules |
||||
{ |
||||
my ($self, $ctx) = @_; |
||||
|
||||
my @moduleList; # module names converted to ksb::Module objects. |
||||
my $optionsRef = $self->{options}; |
||||
|
||||
# Note: This returns a hashref, not a string. |
||||
my $repoSet = $ctx->getOption('git-repository-base'); |
||||
|
||||
# Setup default options for each module |
||||
# If we're in this method, we must be using the git-repository-base method |
||||
# of setting up a module-set, so there is no 'search' or 'ignore' to |
||||
# handle, just create ksb::Module and dump options into them. |
||||
for my $moduleItem ($self->modulesToFind()) { |
||||
my $moduleName = $moduleItem; |
||||
|
||||
$moduleName =~ s/\.git$//; |
||||
|
||||
my $newModule = ksb::Module->new($ctx, $moduleName); |
||||
|
||||
$self->_initializeNewModule($newModule); |
||||
|
||||
push @moduleList, $newModule; |
||||
|
||||
# Setup the only feature actually specific to a module-set, which is |
||||
# the repository handling. |
||||
my $selectedRepo = $repoSet->{$optionsRef->{'repository'}}; |
||||
$newModule->setOption('repository', $selectedRepo . $moduleItem); |
||||
} |
||||
|
||||
if (not scalar @moduleList) { |
||||
warning ("No modules were defined for the module-set $self->name()"); |
||||
warning ("You should use the g[b[use-modules] option to make the module-set useful."); |
||||
} |
||||
|
||||
return @moduleList; |
||||
} |
||||
|
||||
1; |
||||
@ -0,0 +1,222 @@ |
||||
package ksb::ModuleSet::KDEProjects; |
||||
|
||||
# Class: ModuleSet::KDEProjects |
||||
# |
||||
# This represents a collective grouping of modules that share common options, |
||||
# and are obtained via the kde-projects database at |
||||
# https://projects.kde.org/kde_projects.xml |
||||
# |
||||
# See also the parent class ModuleSet, from which most functionality is derived. |
||||
# |
||||
# The only changes here are to allow for expanding out module specifications |
||||
# (except for ignored modules), by using KDEXMLReader. |
||||
# |
||||
# See also: ModuleSet |
||||
|
||||
use strict; |
||||
use warnings; |
||||
use v5.10; |
||||
no if $] >= 5.018, 'warnings', 'experimental::smartmatch'; |
||||
|
||||
our $VERSION = '0.10'; |
||||
our @ISA = qw(ksb::ModuleSet); |
||||
|
||||
use ksb::Module; |
||||
use ksb::Debug; |
||||
use ksb::KDEXMLReader; |
||||
use ksb::BuildContext 0.20; |
||||
use ksb::Util; |
||||
|
||||
# A 'new' subroutine is not needed, ksb::ModuleSet's should do the right thing |
||||
|
||||
# Simple utility subroutine. See List::Util's perldoc |
||||
sub none_true |
||||
{ |
||||
($_ && return 0) for @_; |
||||
return 1; |
||||
} |
||||
|
||||
# Function: getMetadataModule |
||||
# |
||||
# A 'static' method that returns a <Module> that should be included first in |
||||
# the build context's module list. It will be configured to download required |
||||
# updates to the build-metadata required for kde-projects module support. |
||||
# It should only be included exactly once in the build context, if there are |
||||
# one or more ksb::ModuleSet::KDEProjects present in the module list. |
||||
# |
||||
# Parameters: |
||||
# ctx - the <ksb::BuildContext> for this script execution. |
||||
# |
||||
# Returns: The <Module> to added to the beginning of the update. |
||||
sub getMetadataModule |
||||
{ |
||||
my $ctx = assert_isa(shift, 'ksb::BuildContext'); |
||||
|
||||
my $metadataModule = ksb::Module->new($ctx, 'kde-build-metadata'); |
||||
|
||||
# Hardcode the results instead of expanding out the project info |
||||
$metadataModule->setOption('repository', 'kde:kde-build-metadata'); |
||||
$metadataModule->setOption('#xml-full-path', 'kde-build-metadata'); |
||||
$metadataModule->setOption('#branch:stable', 'master'); |
||||
$metadataModule->setScmType('metadata'); |
||||
$metadataModule->setOption('disable-snapshots', 1); |
||||
$metadataModule->setOption('branch', 'master'); |
||||
|
||||
my $moduleSet = ksb::ModuleSet::KDEProjects->new($ctx, '<kde-projects metadata>'); |
||||
$metadataModule->setModuleSet($moduleSet); |
||||
|
||||
# Ensure we only ever try to update source, not build. |
||||
$metadataModule->phases()->phases('update'); |
||||
return $metadataModule; |
||||
} |
||||
|
||||
# Function: _expandModuleCandidates |
||||
# |
||||
# A class method which goes through the modules in our search list (assumed to |
||||
# be found in the kde-projects XML database) and expands them into their |
||||
# equivalent git modules, and returns the fully expanded list. Non kde-projects |
||||
# modules cause an error, as do modules that do not exist at all within the |
||||
# database. |
||||
# |
||||
# *Note*: Before calling this function, the kde-projects database itself must |
||||
# have been downloaded first. Additionally a <Module> handling build support |
||||
# metadata must be included at the beginning of the module list, see |
||||
# getMetadataModule() for details. |
||||
# |
||||
# *Note*: Any modules that are part of a module-set requiring a specific |
||||
# branch, that don't have that branch, are also elided with only a debug |
||||
# message. This allows for building older branches of KDE even when newer |
||||
# modules are eventually put into the database. |
||||
# |
||||
# Parameters: |
||||
# ctx - The <BuildContext> in use. |
||||
# moduleSearchItem - The search description to expand in ksb::Modules. See |
||||
# _projectPathMatchesWildcardSearch for a description of the syntax. |
||||
# |
||||
# Returns: |
||||
# @modules - List of expanded git <Modules>. |
||||
# |
||||
# Throws: |
||||
# Runtime - if the kde-projects database was required but couldn't be |
||||
# downloaded or read. |
||||
# Runtime - if the git-desired-protocol is unsupported. |
||||
# Runtime - if an "assumed" kde-projects module was not actually one. |
||||
sub _expandModuleCandidates |
||||
{ |
||||
my $self = assert_isa(shift, 'ksb::ModuleSet::KDEProjects'); |
||||
my $ctx = assert_isa(shift, 'ksb::BuildContext'); |
||||
my $moduleSearchItem = shift; |
||||
|
||||
my $databaseFile = $ctx->getKDEProjectMetadataFilehandle() or |
||||
croak_runtime("kde-projects repository information could not be downloaded: $!"); |
||||
my $srcdir = $ctx->getSourceDir(); |
||||
|
||||
my $protocol = $ctx->getOption('git-desired-protocol') || 'git'; |
||||
if (!list_has(['git', 'http'], $protocol)) { |
||||
error (" b[y[*] Invalid b[git-desired-protocol] $protocol"); |
||||
error (" b[y[*] Try setting this option to 'git' if you're not using a proxy"); |
||||
croak_runtime ("Invalid git-desired-protocol: $protocol"); |
||||
} |
||||
|
||||
my $xmlReader = ksb::KDEXMLReader->new($databaseFile); |
||||
my @allXmlResults = $xmlReader->getModulesForProject($moduleSearchItem, $protocol); |
||||
|
||||
# It's possible to match modules which are marked as inactive on |
||||
# projects.kde.org, elide those. |
||||
my @xmlResults = grep { $_->{'active'} ne 'false' } (@allXmlResults); |
||||
|
||||
# Bug 307694 |
||||
my $moduleSetBranch = $self->{'options'}->{'branch'} // ''; |
||||
if ($moduleSetBranch) { |
||||
debug ("Filtering kde-projects modules that don't have a $moduleSetBranch branch"); |
||||
@xmlResults = grep { |
||||
list_has($_->{'branches'}, $moduleSetBranch) |
||||
} (@xmlResults); |
||||
} |
||||
|
||||
if (!@xmlResults) { |
||||
warning (" y[b[*] Module y[$moduleSearchItem] is apparently XML-based, but contains no\n" . |
||||
"active modules to build!"); |
||||
my $count = scalar @allXmlResults; |
||||
if ($count > 0) { |
||||
warning ("\tAlthough no active modules are available, there were\n" . |
||||
"\t$count inactive modules. Perhaps the git modules are not ready?"); |
||||
} |
||||
} |
||||
|
||||
# Setup module options. |
||||
my @moduleList; |
||||
my @ignoreList = $self->modulesToIgnore(); |
||||
|
||||
foreach (@xmlResults) { |
||||
my $result = $_; |
||||
my $repo = $result->{'repo'}; |
||||
|
||||
# Prefer kde: alias to normal clone URL. |
||||
$repo =~ s(^git://anongit\.kde\.org/)(kde:); |
||||
|
||||
my $newModule = ksb::Module->new($ctx, $result->{'name'}); |
||||
$self->_initializeNewModule($newModule); |
||||
$newModule->setOption('repository', $repo); |
||||
$newModule->setOption('#xml-full-path', $result->{'fullName'}); |
||||
$newModule->setOption('#branch:stable', $result->{'branch:stable'}); |
||||
|
||||
my $tarball = $result->{'tarball'}; |
||||
$newModule->setOption('#snapshot-tarball', $tarball) if $tarball; |
||||
|
||||
if (none_true( |
||||
map { |
||||
ksb::KDEXMLReader::_projectPathMatchesWildcardSearch( |
||||
$result->{'fullName'}, |
||||
$_ |
||||
) |
||||
} (@ignoreList))) |
||||
{ |
||||
push @moduleList, $newModule; |
||||
} |
||||
else { |
||||
debug ("--- Ignoring matched active module $newModule in module set " . |
||||
$self->name()); |
||||
} |
||||
}; |
||||
|
||||
return @moduleList; |
||||
} |
||||
|
||||
# This function should be called after options are read and build metadata is |
||||
# available in order to convert this module set to a list of ksb::Module. |
||||
# Any modules ignored by this module set are excluded from the returned list. |
||||
# The modules returned have not been added to the build context. |
||||
sub convertToModules |
||||
{ |
||||
my ($self, $ctx) = @_; |
||||
|
||||
my @moduleList; # module names converted to ksb::Module objects. |
||||
|
||||
# Setup default options for each module |
||||
# Extraction of relevant XML modules will be handled immediately after |
||||
# this phase of execution. |
||||
for my $moduleItem ($self->modulesToFind()) { |
||||
# eval in case the XML processor throws an exception. |
||||
undef $@; |
||||
my @candidateModules = eval { |
||||
$self->_expandModuleCandidates($ctx, $moduleItem); |
||||
}; |
||||
|
||||
if ($@) { |
||||
die $@ if ref $@; # Forward exception objects up |
||||
croak_runtime("The XML for the KDE Project database could not be understood: $@"); |
||||
} |
||||
|
||||
push @moduleList, @candidateModules; |
||||
} |
||||
|
||||
if (not scalar @moduleList) { |
||||
warning ("No modules were defined for the module-set $self->name()"); |
||||
warning ("You should use the g[b[use-modules] option to make the module-set useful."); |
||||
} |
||||
|
||||
return @moduleList; |
||||
} |
||||
|
||||
1; |
||||
@ -0,0 +1,33 @@ |
||||
package ksb::ModuleSet::Null; |
||||
|
||||
# Class: ModuleSet::Null |
||||
# |
||||
# Used automatically by <Module> to represent the abscence of a <ModuleSet> without |
||||
# requiring definedness checks. |
||||
|
||||
use strict; |
||||
use warnings; |
||||
use v5.10; |
||||
|
||||
our $VERSION = '0.10'; |
||||
our @ISA = qw(ksb::ModuleSet); |
||||
|
||||
use ksb::Util; |
||||
|
||||
sub new |
||||
{ |
||||
my $class = shift; |
||||
return bless {}, $class; |
||||
} |
||||
|
||||
sub name |
||||
{ |
||||
return ''; |
||||
} |
||||
|
||||
sub convertToModules |
||||
{ |
||||
croak_internal("kdesrc-build should not have made it to this call. :-("); |
||||
} |
||||
|
||||
1; |
||||
Loading…
Reference in new issue