diff --git a/kdesrc-build b/kdesrc-build index 03671c9..ead9bcd 100755 --- a/kdesrc-build +++ b/kdesrc-build @@ -49,8 +49,7 @@ use File::Find; # For our lndir reimplementation. use File::Path qw(remove_tree); use File::Glob ':glob'; use File::Spec; # tmpdir, rel2abs -use File::Temp qw(tempfile); -use List::Util qw(first); +use List::Util qw(first min); use LWP::UserAgent; use URI; # For git-clone snapshot support use Sys::Hostname; @@ -67,9 +66,11 @@ use ksb::IPC::Pipe 0.20; use ksb::IPC::Null; use ksb::KDEXMLReader; use ksb::Updater::Git; -use ksb::BuildContext; +use ksb::BuildContext 0.20; use ksb::RecursiveFH; use ksb::Module; +use ksb::ModuleSet; +use ksb::ModuleSet::KDEProjects; use ksb::DependencyResolver 0.20; use 5.010_000; # Require Perl 5.10.0 @@ -628,101 +629,26 @@ EOF return $module; } -# Tries to download the kde_projects.xml file needed to make XML module support -# work. Only tries once per script run. If it does succeed, the result is saved -# to $srcdir/kde_projects.xml -# -# Returns the file handle that the database can be retrieved from. May throw an -# exception if an error occurred. -sub ensure_projects_xml_present -{ - my $ctx = assert_isa(shift, 'ksb::BuildContext'); - - state $file; - state $cachedSuccess; - - # See if we've already tried to download. If we ever try to download for - # real, we end up unlinking the file if it didn't successfully complete the - # download, so we shouldn't have to worry about a corrupt XML file hanging - # out for all time. - if (defined $cachedSuccess && !$cachedSuccess) { - croak_internal("Attempted to find projects.xml after it already failed"); - } - - if ($cachedSuccess) { - open my $fh, '<', $file or die make_exception('Runtime', "Unable to open $file: $!"); - return $fh; - } - - # Not previously attempted, let's make a try. - my $srcdir = $ctx->getSourceDir(); - my $fileHandleResult; - - super_mkdir($srcdir) unless -d "$srcdir"; - $file = "$srcdir/kde_projects.xml"; - my $url = "http://projects.kde.org/kde_projects.xml"; - - my $result = 1; - - # Must use ->phases() directly to tell if we will be updating since - # modules are not all processed until after this function is called... - my $updating = grep { /^update$/ } (@{$ctx->phases()}); - if (!pretending() && $updating) { - info (" * Downloading projects.kde.org project database..."); - $result = download_file($url, $file, $ctx->getOption('http-proxy')); - } - elsif (! -e $file) { - note (" * Downloading projects.kde.org project database (will not be saved in pretend mode)..."); - - # Unfortunately dumping the HTTP output straight to the XML parser is a - # wee bit more complicated than I feel like dealing with => use a temp - # file. - (undef, $file) = tempfile('kde_projectsXXXXXX', - SUFFIX=>'.xml', TMPDIR=>1, UNLINK=>0); - $result = download_file($url, $file, $ctx->getOption('http-proxy')); - open ($fileHandleResult, '<', $file) or croak_runtime("Unable to open KDE Project database $file: $!"); - } - else { - info (" * y[Using existing projects.kde.org project database], output may change"); - info (" * when database is updated next."); - } - - $cachedSuccess = $result; - - if (!$result) { - unlink $file if -e $file; - croak_runtime("Unable to download kde_projects.xml for the kde-projects repository!"); - } - - if (!$fileHandleResult) { - open ($fileHandleResult, '<', $file) or die - make_exception('Runtime', "Unable to open $file: $!"); - } - - return $fileHandleResult; -} - # Reads in a "moduleset". # # First parameter is the filehandle to the config file to read from. # Second parameter is the name of the moduleset, which is really the name # of the base repository to use. -# Returns the expanded list of module names to include. +# Returns a ksb::ModuleSet describing the module-set encountered, which may +# need to be further expanded (see ksb::ModuleSet::convertToModules). sub parse_moduleset { my $ctx = assert_isa(shift, 'ksb::BuildContext'); my $fileReader = shift; my $moduleSetName = shift || ''; - my $repoSet = $ctx->getOption('git-repository-base'); my $rcfile = $ctx->rcFile(); - my @modules; - my %ignoredModules; - my %optionSet; # We read all options, and apply them to all modules my $startLine = $.; # For later error messages + my $internalModuleSetName = + $moduleSetName || ""; - # Used to strip leading parts of module name if no wildcard present - my $pathLeadersRE = qr,^.*/(?!\*),; + my $moduleSet = ksb::ModuleSet->new($ctx, $internalModuleSetName); + my %optionSet; # We read all options, and apply them to all modules while($_ = readNextLogicalLine($fileReader)) { last if /^end\s+module(-?set)?$/; @@ -730,24 +656,26 @@ sub parse_moduleset my ($option, $value) = split_option_value($ctx, $_); if ($option eq 'use-modules') { - @modules = split(' ', $value); + my @modules = split(' ', $value); if (not @modules) { error ("No modules were selected for the current module-set"); error ("in the y[use-modules] on line $. of $rcfile"); die make_exception('Config', 'Invalid use-modules'); } + + $moduleSet->setModulesToFind(@modules); } elsif ($option eq 'ignore-modules') { my @modulesToIgnore = split(' ', $value); - s,$pathLeadersRE,, foreach @modulesToIgnore; - @ignoredModules{@modulesToIgnore} = (1) x @modulesToIgnore; if (not @modulesToIgnore) { error ("No modules were selected for the current module-set"); error ("in the y[ignore-modules] on line $. of $rcfile"); die make_exception('Config', 'Invalid ignore-modules'); } + + $moduleSet->setModulesToIgnore(@modulesToIgnore); } elsif ($option eq 'set-env') { Module::processSetEnvOption(\%optionSet, $option, $value); @@ -757,6 +685,8 @@ sub parse_moduleset } } + $moduleSet->setOptions(\%optionSet); + # Check before we use this module set whether the user did something silly. my $repoSet = $ctx->getOption('git-repository-base'); if (!exists $optionSet{'repository'}) { @@ -777,8 +707,8 @@ EOF not exists $repoSet->{$optionSet{'repository'}}) { my $projectID = KDE_PROJECT_ID; - my $moduleSetId = "module-set"; - $moduleSetId = "module-set ($moduleSetName)" if $moduleSetName; + my $moduleSetId = $moduleSetName ? "module-set ($moduleSetName)" + : "module-set"; error (< "libs", which would be read - # in as *all* "*/libs/*" by KDEXMLReader later. - my $newModule = ksb::Module->new($ctx, $moduleName); - $newModule->setModuleSet($moduleSetName); - $newModule->setScmType($usingXML ? 'proj' : 'git'); - $newModule->setOption('#xml-search-path', $moduleFullName) if $usingXML; - push @moduleList, $newModule; - - # Dump all options into the existing ksb::Module's options. - $newModule->setOption(%optionSet); - - # Fixup for the special repository handling if need be. - if (!$usingXML && exists $optionSet{'repository'}) { - $selectedRepo = $repoSet->{$optionSet{'repository'}} unless $selectedRepo; - $newModule->setOption('repository', $selectedRepo . $moduleName); - } - } - - $ctx->addToIgnoreList(keys %ignoredModules); - - if (not scalar @moduleList) { - warning ("No modules were defined for the module-set in r[b[$rcfile] starting at line y[b[$startLine]"); - warning ("You should use the g[b[use-modules] option to make the module-set useful."); + if ($optionSet{'repository'} eq KDE_PROJECT_ID) { + # Perl-specific note! re-blessing the module set into the right 'class' + # You'd probably have to construct an entirely new object and copy the + # members over in other languages. + bless $moduleSet, 'ksb::ModuleSet::KDEProjects'; } - return @moduleList; + return $moduleSet; } -# Function: expandXMLModules +# Function: expandModuleSets # -# Goes through the provided modules that have the 'proj' type (i.e. XML -# projects.kde.org database) and expands the proj-types into their equivalent -# git modules, and returns the fully expanded list. Non-proj modules are -# included in the sequence they were originally. +# Replaces in an input list from the command line that name +# module-sets listed in the configuration file, and returns the new list. # -# *Note*: This function will trigger network download of the kde-projects -# database if any relevant modules are identified. In addition, a build-support -# module will be included in this case, which requires special care. +# are ignored if found in the input list, and transferred to the +# output list in the same relative order. # -# *Note*: Any modules that are part of a module-set requiring a specific -# branch, that don't have that branch, are also elided with only a debug -# message. This allows for building older branches of KDE even when newer -# modules are eventually put into the database. +# This function may result in kde-projects metadata being downloaded and +# processed. # # Parameters: -# ctx - The in use. -# @modules - The list of to perform proj-expansion on. +# $ctx - in use for this script execution. +# @modules - list of , to be expanded. # # Returns: -# A tuple of: -# ($metadataModule, @modules) -# $metadataModule - If the build-support repository was included this will -# be a , with an SCM type of . -# Many other support functions will require this metadata be updated -# first. If metadata is *not* needed this will be undef. -# @modules - List of remaining expanded git . -# -# Throws: -# Runtime - if the kde-projects database was required but couldn't be -# downloaded or read. -# Runtime - if the git-desired-protocol is unsupported. -# Runtime - if an "assumed" kde-projects module was not actually one. -sub expandXMLModules +# $metadataModule - a to use if needed for kde-projects support, can be +# undef if not actually required this run. +# @modules - List of with any module-sets expanded into . +sub expandModuleSets { - my $ctx = assert_isa(shift, 'ksb::BuildContext'); - my @modules = @_; + my ($ctx, @buildModuleList) = @_; my $metadataModule; - # If we detect a KDE project we want to also pull in a "build support" - # repository that will contain metadata such as which modules depend on - # which other ones, what modules shouldn't be built, etc. - my $repositoryMetadataRequired = 0; + if (grep { $_->isa('ksb::ModuleSet::KDEProjects') } @buildModuleList) { + debug ("Introducing metadata module into the build"); + $metadataModule = ksb::ModuleSet::KDEProjects::getMetadataModule($ctx); + assert_isa($metadataModule, 'ksb::Module'); + } - # Using a sub allows me to use the 'return' keyword. my $filter = sub { - my $moduleSet = shift; - - # Only attempt to expand out XML-based modules. - return $moduleSet if !$moduleSet->scm()->isa('ksb::Updater::KDEProject'); - - my $databaseFile = ensure_projects_xml_present($ctx) or - croak_runtime("kde-projects repository information could not be downloaded: $!"); - - # At this point we know we'll need the kde-build-metadata module, force - # it in by setting a flag to be used by the higher-level subroutine. - $repositoryMetadataRequired = 1; - - my $name = $moduleSet->getOption('#xml-search-path') || - $moduleSet->name(); # Best-guess fallback - my $srcdir = $ctx->getSourceDir(); - - my $xmlReader = ksb::KDEXMLReader->new($databaseFile); - my $protocol = $ctx->getOption('git-desired-protocol') || 'git'; - - if (!list_has(['git', 'http'], $protocol)) { - error (" b[y[*] Invalid b[git-desired-protocol] $protocol"); - error (" b[y[*] Try setting this option to 'git' if you're not using a proxy"); - croak_runtime ("Invalid git-desired-protocol: $protocol"); - } - - my @allXmlResults = $xmlReader->getModulesForProject($name, $protocol); - - # It's possible to match modules which are marked as inactive on - # projects.kde.org, elide those. - my @xmlResults = grep { $_->{'active'} ne 'false' } (@allXmlResults); + my $moduleOrSetName = $_->name(); - # Bug 307694 - my $moduleSetBranch = $moduleSet->getOption('branch'); - if ($moduleSetBranch) { - debug ("Filtering kde-projects modules that don't have a $moduleSetBranch branch"); - @xmlResults = grep { - list_has($_->{'branches'}, $moduleSetBranch) - } (@xmlResults); + # 'proj' module types can only come from command line -- we assume the + # user is trying to build a module from the kde-projects repo without + # first putting into rc-file. + if ($_->isa('ksb::Module') && $_->scmType() ne 'proj') { + return $_; } - if (!@xmlResults) { - # If this was a "guessed XML module" then we guessed wrong, and it's really - # a misspelling. - if ($moduleSet->getOption('#guessed-kde-project')) { - croak_runtime("Unknown module or module-set: $name"); - } - - warning (" y[b[*] Module y[$name] is apparently XML-based, but contains no\n" . - "active modules to build!"); - my $count = scalar @allXmlResults; - if ($count > 0) { - warning ("\tAlthough no active modules are available, there were\n" . - "\t$count inactive modules. Perhaps the git modules are not ready?"); - } + if ($_->isa('ksb::ModuleSet')) { + return $_->convertToModules($ctx); } - # Setup module options. This alters the results in @xmlResults. - foreach (@xmlResults) { - my $result = $_; - my $repo = $result->{'repo'}; - - # Prefer kde: alias to normal clone URL. - $repo =~ s(^git://anongit\.kde\.org/)(kde:); - - # This alters the item we were looking at. - $_ = ksb::Module->new($ctx, $result->{'name'}); - $_->setScmType('git'); - $_->cloneOptionsFrom($moduleSet); - $_->phases()->phases($moduleSet->phases()->phases()); # Copy phases over - $_->setModuleSet($moduleSet->moduleSet()); - $_->setOption('repository', $repo); - $_->setOption('#xml-full-path', $result->{'fullName'}); - $_->setOption('#branch:stable', $result->{'branch:stable'}); - - my $tarball = $result->{'tarball'}; - $_->setOption('#snapshot-tarball', $tarball) if $tarball; - }; - - return @xmlResults; - }; - - # eval in case the XML processor throws an exception. - undef $@; - my @results = eval { map { &$filter($_) } (@modules) }; - - if ($@) { - die $@ if ref $@; # Forward exception objects up - croak_runtime("The XML for the KDE Project database could not be understood: $@"); - } - elsif ($repositoryMetadataRequired) { - $metadataModule = ksb::Module->new($ctx, 'kde-build-metadata'); - - # Hardcode the results instead of expanding out the project info - $metadataModule->setOption('repository', 'kde:kde-build-metadata'); - $metadataModule->setOption('#xml-full-path', 'kde-build-metadata'); - $metadataModule->setOption('#branch:stable', 'master'); - $metadataModule->setScmType('metadata'); - $metadataModule->setOption('disable-snapshots', 1); - $metadataModule->setOption('branch', 'master'); - - # Ensure we only ever try to update source, not build. - $metadataModule->phases()->phases('update'); - } - - return ($metadataModule, @results); -} - -# Function: expandModuleSets -# -# Replaces in an input list from the command line that name -# module-sets listed in the configuration file, and returns the new list. -# -# A is assumed to name a module-set if all of the following are true. -# * No options were read in for it. -# * The module isn't already forced to be a kde-project module (as can happen -# on the command line). -# * from the rcFileModules were included in a module-set of the same -# name. -# -# Any that meets these conditions is replaced by all of the matching -# module-set in the return list. -# -# Any that doesn't match a known rcFile module *or* module-set is -# assumed to be a kde-project module and altered appropriately. -# -# Parameters: -# $@modules - listref of to be expanded. -# $@rcFileModules - listref of that list modules read in from the -# configuration file. -# -# Returns: -# @modules - List of with any module-sets expanded into . -# No kde-project module expansion occurs. -sub expandModuleSets -{ - my ($buildModuleList, $knownModules) = @_; - - my $filter = sub { - my $setName = $_->name(); - - # If the module name matches a read-in ksb::Module, then it's not a set. - return $_ if grep { $setName eq $_->name() } (@$knownModules); - - # XML module can only happen if forced by user on command line, allow - # it. - return $_ if $_->scmType() eq 'proj'; - - # Likewise with l10n module. - return $_ if $_->scmType() eq 'l10n'; - - # Otherwise assume it's a set, replace this with all sub-modules in that - # module set. - my @modulesInSet = grep - { ($_->moduleSet() // '') eq $setName } - (@$knownModules); - - if (!@modulesInSet) { - # If we make it to this point the module is either completely - # unknown, or possibly part of a kde-projects module-set (it can't - # be part of a regular module-set as those modules are already in - # @knownModules). To allow things to continue we will - # optimistically mark the module as a kde-projects module and then - # cross our fingers... - $_->setScmType('proj'); - $_->setOption('#guessed-kde-project', 1); - push @modulesInSet, $_; - } + my $moduleSet = ksb::ModuleSet::KDEProjects->new($ctx, ''); + $moduleSet->setModulesToFind($_->name()); + $moduleSet->{options}->{'#guessed-kde-project'} = 1; - return @modulesInSet; + debug ("--- Trying to find a home for $_"); + return $moduleSet->convertToModules($ctx); }; - return map { &$filter } (@$buildModuleList); + return ($metadataModule, map { &$filter } (@buildModuleList)); } # Function: read_options @@ -1085,9 +805,9 @@ sub expandModuleSets # methods (e.g. subclass). # # Returns: -# @module - List of defined in the configuration file. Note that -# although any module-sets found will have been expanded out, no kde-projects -# modules will have been further expanded out. +# @module - Heterogenous list of and defined in the +# configuration file. No module sets will have been expanded out (either +# kde-projects or standard sets). # # Throws: # - Config exceptions. @@ -1155,7 +875,7 @@ sub read_options if (my @modules = grep { $_->name() eq $modulename } @module_list) { # We check for definedness as a module-set can exist but be # unnamed. - if (!defined $modules[0]->moduleSet()) { + if ($modules[0]->moduleSet()->isa('ksb::ModuleSet::Null')) { warning ("Multiple module declarations for $modules[0]"); } @@ -1692,7 +1412,14 @@ DONE $ctx->setOption('#async', 0); } - return map { ksb::Module->new($ctx, $_) } (@enteredModules); + return map { + my $module = ksb::Module->new($ctx, $_); + # Following will be replaced by option modules if present in rc-file. + $module->setScmType('proj'); + $module->setOption('#guessed-kde-project', 1); + $module->phases()->phases($phases->phases()); + $module; + } (@enteredModules); } sub updateModulePhases @@ -2269,8 +1996,8 @@ EOF while (my $module = shift @modules) { - my $moduleSet = $module->moduleSet() // ''; my $moduleName = $module->name(); + my $moduleSet = $module->moduleSet()->name(); my $modOutput = "$module"; if (debugging(ksb::Debug::WHISPER)) { @@ -2514,12 +2241,12 @@ sub handle_uninstall # # Parameters: # ctx - in use. -# @modules - List of to apply filters on. +# @modules - List of or to apply filters on. # # Returns: -# list of with any inclusion/exclusion filters applied. Do not -# assume this list will be a strict subset of the input list, however the -# order will not change amongst the input modules. +# list of or with any inclusion/exclusion filters +# applied. Do not assume this list will be a strict subset of the input list, +# however the order will not change amongst the input modules. sub applyModuleFilters { my $ctx = assert_isa(shift, 'ksb::BuildContext'); @@ -2570,19 +2297,11 @@ EOF for (my $i = 0; $i < scalar @moduleList; $i++) { my $module = $moduleList[$i]; - my $moduleSet = $module->moduleSet() // ''; - # If a match was found we can only still be here if looking at - # --resume-after. If we matched on module set name then eliminate - # all of the module set before stopping. - if ($found && ($moduleSet ne $resumePoint)) { - $startIndex = $i; - last; - } - - $found = $module->{'name'} eq $resumePoint || $moduleSet eq $resumePoint; - if ($found && $filterInclusive) { - $startIndex = $i; + $found = $module->name() eq $resumePoint; + if ($found) { + $startIndex = $filterInclusive ? $i : $i + 1; + $startIndex = min($startIndex, scalar @moduleList - 1); last; } } @@ -2605,9 +2324,8 @@ EOF for (my $i = $startIndex; $i < scalar @moduleList; $i++) { my $module = $moduleList[$i]; - my $moduleSet = $module->moduleSet() // ''; - $found = $module->{'name'} eq $stopPoint || $moduleSet eq $stopPoint; + $found = $module->name() eq $stopPoint; if ($found) { $stopIndex = $i - ($filterInclusive ? 1 : 0); last; @@ -2634,8 +2352,9 @@ EOF # # Parameters: # @$modules - Listref of modules to potentially splice in replacements of. -# @$optionModules - Listref to list of the "option" modules, which should be -# of the same level of kde-project expansion as @$modules. +# @$optionModules - Listref to list of the "option" modules (and module-sets), +# which should be of the same level of kde-project expansion as @$modules. A +# module-set might be spliced in to replace a named module. # # Returns: # Nothing. @@ -2645,6 +2364,7 @@ sub spliceOptionModules for (my $i = 0; $i < scalar @{$modulesRef}; $i++) { my $module = ${$modulesRef}[$i]; + my ($optionModule) = grep { $_->name() eq $module->name() } @{$optionModulesRef}; @@ -2954,8 +2674,11 @@ eval $ctx = ksb::BuildContext->new(); my $pendingOptions = { }; + debug ("+++ Reached pre-init phase"); + # Process --help, --install, etc. first. my @modules = process_arguments($ctx, $pendingOptions, @ARGV); + debug ("--- Arguments read: ", scalar @modules, " command line modules"); # Output time once we know if pretending or not. my $time = localtime; @@ -2974,14 +2697,18 @@ eval my $fh = $ctx->loadRcFile(); # If we're still here, read the options - my @optionModules = read_options($ctx, $fh); + my @optionModulesAndSets = read_options($ctx, $fh); close $fh; + debug ("--- Config file read: ", scalar @optionModulesAndSets, " modules/module sets."); + debug (" --- " , (scalar grep { $_->isa('ksb::ModuleSet') } @optionModulesAndSets) , " are module sets."); # Modify the options read from the rc-file to have the pending changes from # the command line. foreach my $pendingModule (keys %{$pendingOptions}) { my $options = ${$pendingOptions}{$pendingModule}; - my ($module) = grep { $pendingModule eq $_->name() } (@optionModules); + my ($module) = grep { + $_->isa('ksb::Module') && $pendingModule eq $_->name() + } (@optionModulesAndSets); if (!$module) { warning ("Tried to set options for unknown module b[y[$pendingModule]"); @@ -2989,7 +2716,6 @@ eval } while (my ($key, $value) = each %{$options}) { - debug ("Setting pending option $key to $value for $pendingModule"); $module->setOption($key, $value); } } @@ -3010,84 +2736,104 @@ eval # Allow named module-sets to be given on the command line. if ($commandLineModules) { - # Copy ksb::Module objects from the ones created by read_options - # since their module-type will actually be set. - spliceOptionModules(\@modules, \@optionModules); + # Copy ksb::Module and ksb::ModuleSet objects from the ones created by + # read_options since their module-type will actually be set. + spliceOptionModules(\@modules, \@optionModulesAndSets); + debug ("--- spliced rc-file modules/sets into command line modules"); + debug (" --- now ", scalar @modules, " modules present"); + + # After this splicing, the only bare ksb::Modules with a 'proj' + # scm type should be the ones from the command line that we got from + # process_arguments. # Modify l10n module inline, if present. for (@modules) { - if ($_->name() eq 'l10n') { $_->setScmType('l10n') } + if ($_->name() eq 'l10n' && $_->isa('ksb::Module')) { + $_->setScmType('l10n') + } } # Filter --resume-foo first so entire module-sets can be skipped. # Wrap in eval to catch runtime errors eval { @modules = applyModuleFilters($ctx, @modules); }; - @modules = expandModuleSets(\@modules, \@optionModules); + + debug ("--- Applied command-line --resume-foo pass, ", scalar @modules, " remain."); + debug (" --- " , (scalar grep { $_->isa("ksb::ModuleSet") } @modules) , " are module sets."); + + ($metadataModule, @modules) = expandModuleSets($ctx, @modules); + + debug ("--- Expanded command-line module sets into ", scalar @modules, " modules."); + debug (" --- Metadata module needed? ", $metadataModule ? "Yes" : "No"); + + # If we have any 'guessed' modules they came from the command line. We + # want them to use ksb::Modules that came from read_options() instead + # of ones from process_arguments(), if any are available. But we don't + # expand module-sets from read_options unconditionally, only ones where + # the name matched between the command line and rc-file, so the + # previous spliceOptionModules might not have found them. (e.g. if you + # named a module on the command line that isn't named directly in the + # rc file but would eventually be found implicitly). To make this work + # we have to expand out our rc-file Modules and try splicing again. + if (first { $_->getOption('#guessed-kde-project', 'module') } @modules) { + my (undef, @expandedOptionModules) = expandModuleSets($ctx, @optionModulesAndSets); + spliceOptionModules(\@modules, \@expandedOptionModules); + + debug ("--- re-spliced rc-file modules/sets into command line modules"); + debug (" --- now ", scalar @modules, " modules present"); + } + + # At this point @modules has no more module sets, kde-projects or + # otherwise. ksb::Module->setModuleSource('cmdline'); } else { - @modules = @optionModules; + # Build everything in the rc-file, in the order specified. + ($metadataModule, @modules) = expandModuleSets($ctx, @optionModulesAndSets); + + debug ("--- Expanded rc-file module sets into ", scalar @modules, " modules."); + debug (" --- Metadata module needed? ", $metadataModule ? "Yes" : "No"); if ($ctx->getOption('kde-languages')) { my $l10nModule = ksb::Module->new($ctx, 'l10n'); $l10nModule->setScmType('l10n'); $l10nModule->setBuildSystem($l10nModule->scm()); + debug ("--- Added l10n module to end of build"); push @modules, $l10nModule; } ksb::Module->setModuleSource('config'); } - # Must be done before filtering so that we can filter under module-sets. - ($metadataModule, @modules) = expandXMLModules($ctx, @modules); - - # Fixup any "guessed" kde-projects modules. If we have a guessed module we - # know that a kde-project expansion pass was actually performed, but we - # don't know that the config-file option-module was found since we never - # ran a kde-project expansion pass on the optionModules (doing this - # unconditionally would be waste for runs where we know exactly what repos - # to build, although we're quickly approaching the day where that won't - # matter). - if (my @guessed = grep { $_->getOption('#guessed-kde-project') } @modules) { - whisper (" * Performing second-level kde-project expansion for cmdline modules"); - my $newMetadata; - ($newMetadata, @optionModules) = expandXMLModules($ctx, @optionModules); - spliceOptionModules(\@guessed, \@optionModules); - - # Metadata module if present should be the same, just pick either. - $metadataModule = $metadataModule // $newMetadata; - - # Now that we have fixed the @guessed entries, we still have to merge - # those back in. - my $i = 0; - for my $fixedModule (@guessed) { - $i = first { $modules[$_]->name() eq $fixedModule->name() } ($i .. $#modules); - splice (@modules, $i, 1, $fixedModule) if defined $i; - $i = 0 if not defined $i; # Reset when no match found - } - } - # Filter --resume-foo options. This might be a second pass, but that should # be OK since there's nothing different going on from the first pass in that # event. @modules = applyModuleFilters($ctx, @modules); + debug ("--- Applied all-modules --resume-foo pass, ", scalar @modules, " remain."); + debug (" --- ", (scalar grep { $_->isa("ksb::ModuleSet") } @modules), " are module sets (should be 0!)."); + # Apply kde-languages, by appending needed l10n modules to the end of the # build. @modules = expandl10nModules($ctx, @modules); + debug ("--- Expanded l10n modules. ", scalar @modules, " remain."); + # Remove ignored modules and/or module-sets @modules = grep { - not exists $ignore_list{$_->name()} && not exists $ignore_list{$_->moduleSet()} + not exists $ignore_list{$_->name()} && not exists $ignore_list{$_->moduleSet()->name()} } (@modules); + debug ("--- Handled ignore lists. ", scalar @modules, " remain."); + # If modules were on the command line then they are effectively forced to # process unless overridden by command line options as well. If phases # *were* overridden on the command line, then no update pass is required # (all modules already have correct phases) @modules = updateModulePhases(@modules) unless $commandLineModules; + debug ("--- Updated module phases. ", scalar @modules, " remain."); + if (exists $ENV{KDESRC_BUILD_DUMP_CONTEXT}) { local $Data::Dumper::Indent = 1; local $Data::Dumper::Sortkeys = 1; @@ -3112,6 +2858,8 @@ eval # execution phase {{{ $ctx->loadPersistentOptions(); + debug ("+++ Reached execution phase"); + # If we have kde-build-metadata we must process it first, ASAP. if ($metadataModule) { eval { diff --git a/modules/ksb/BuildContext.pm b/modules/ksb/BuildContext.pm index a281d2d..e7f8d84 100644 --- a/modules/ksb/BuildContext.pm +++ b/modules/ksb/BuildContext.pm @@ -10,7 +10,7 @@ use warnings; use v5.10; no if $] >= 5.018, 'warnings', 'experimental::smartmatch'; -our $VERSION = '0.10'; +our $VERSION = '0.20'; use Carp 'confess'; use File::Basename; # dirname @@ -23,6 +23,7 @@ use ksb::Util; use ksb::PhaseList; use ksb::Module; use ksb::Version qw(scriptVersion); +use File::Temp qw(tempfile); # We derive from ksb::Module so that BuildContext acts like the 'global' # ksb::Module, with some extra functionality. @@ -133,6 +134,7 @@ sub new rcFile => undef, env => { }, ignore_list => [ ], # List of XML paths to ignore completely. + kde_projects_filehandle => undef, # Filehandle to read database from. ); # Merge all new options into our self-hash. @@ -848,4 +850,68 @@ sub setPersistentOption $persistent_opts->{$moduleName}{$key} = $value; } +# Tries to download the kde_projects.xml file needed to make XML module support +# work. Only tries once per script run. If it does succeed, the result is saved +# to $srcdir/kde_projects.xml +# +# Returns the file handle that the database can be retrieved from. May throw an +# exception if an error occurred. +sub getKDEProjectMetadataFilehandle +{ + my $self = assert_isa(shift, 'ksb::BuildContext'); + + # Return our current filehandle if one exists. + if (defined $self->{kde_projects_filehandle}) { + my $fh = $self->{kde_projects_filehandle}; + $fh->seek(0, 0); # Return to start + return $fh; + } + + # Not previously attempted, let's make a try. + my $srcdir = $self->getSourceDir(); + my $fileHandleResult = IO::File->new(); + + super_mkdir($srcdir) unless -d "$srcdir"; + my $file = "$srcdir/kde_projects.xml"; + my $url = "http://projects.kde.org/kde_projects.xml"; + + my $result = 1; + + # Must use ->phases() directly to tell if we will be updating since + # modules are not all processed until after this function is called... + my $updating = grep { /^update$/ } (@{$self->phases()}); + if (!pretending() && $updating) { + info (" * Downloading projects.kde.org project database..."); + $result = download_file($url, $file, $self->getOption('http-proxy')); + } + elsif (! -e $file) { + note (" * Downloading projects.kde.org project database (will not be saved in pretend mode)..."); + + # Unfortunately dumping the HTTP output straight to the XML parser is a + # wee bit more complicated than I feel like dealing with => use a temp + # file. + (undef, $file) = tempfile('kde_projectsXXXXXX', + SUFFIX=>'.xml', TMPDIR=>1, UNLINK=>0); + $result = download_file($url, $file, $self->getOption('http-proxy')); + open ($fileHandleResult, '<', $file) or croak_runtime("Unable to open KDE Project database $file: $!"); + } + else { + info (" * y[Using existing projects.kde.org project database], output may change"); + info (" * when database is updated next."); + } + + if (!$result) { + unlink $file if -e $file; + croak_runtime("Unable to download kde_projects.xml for the kde-projects repository!"); + } + + if (!$fileHandleResult->opened()) { + open ($fileHandleResult, '<', $file) or die + make_exception('Runtime', "Unable to open $file: $!"); + } + + $self->{kde_projects_filehandle} = $fileHandleResult; + return $fileHandleResult; +} + 1; diff --git a/modules/ksb/KDEXMLReader.pm b/modules/ksb/KDEXMLReader.pm index 7429ae4..2f03d6d 100644 --- a/modules/ksb/KDEXMLReader.pm +++ b/modules/ksb/KDEXMLReader.pm @@ -112,26 +112,14 @@ sub xmlTagStart if (exists $xmlGroupingIds{$element}) { push @nameStack, $attrs{'identifier'}; - # If we're not tracking something, see if we should be. The logic is - # fairly long-winded but essentially just breaks searchProject into - # its components and compares it item-for-item to the end of our name - # stack. + # If we're not tracking something, see if we should be. if ($trackingReposFlag <= 0) { - my @searchParts = split(m{/}, $searchProject); - if (scalar @nameStack >= scalar @searchParts) { - my @candidateArray = @nameStack[-(scalar @searchParts)..-1]; - die "candidate vs. search array mismatch" if $#candidateArray != $#searchParts; - - $trackingReposFlag = 1; - for (my $i = 0; $i < scalar @searchParts; ++$i) { - if (($searchParts[$i] ne $candidateArray[$i]) && - ($searchParts[$i] ne '*')) - { - $trackingReposFlag = 0; - last; - } - } - } + $trackingReposFlag = + _projectPathMatchesWildcardSearch( + join('/', @nameStack), $searchProject + ) + ? 1 + : 0; } } @@ -237,4 +225,49 @@ sub xmlCharData } } +# Utility subroutine, returns true if the given kde-project full path (e.g. +# kde/kdelibs/nepomuk-core) matches the given search item. +# +# The search item itself is based on path-components. Each path component in +# the search item must be present in the equivalent path component in the +# module's project path for a match. A '*' in a path component position for the +# search item matches any project path component. +# +# Finally, the search is pinned to search for a common suffix. E.g. a search +# item of 'kdelibs' would match a project path of 'kde/kdelibs' but not +# 'kde/kdelibs/nepomuk-core'. However 'kdelibs/*' would match +# 'kde/kdelibs/nepomuk-core'. +# +# First parameter is the full project path from the kde-projects database. +# Second parameter is the search item. +# Returns true if they match, false otherwise. +sub _projectPathMatchesWildcardSearch +{ + my ($projectPath, $searchItem) = @_; + + my @searchParts = split(m{/}, $searchItem); + my @nameStack = split(m{/}, $projectPath); + + if (scalar @nameStack >= scalar @searchParts) { + # This logic is fairly long-winded but essentially just breaks + # searchItem into its components and compares it item-for-item to + # the end of our name stack. + my @candidateArray = @nameStack[-(scalar @searchParts)..-1]; + die "candidate vs. search array mismatch" if $#candidateArray != $#searchParts; + + for (my $i = 0; $i < scalar @searchParts; ++$i) { + if (($searchParts[$i] ne $candidateArray[$i]) && + ($searchParts[$i] ne '*')) + { + return; + } + } + } + else { + return; + } + + return 1; +} + 1; diff --git a/modules/ksb/Module.pm b/modules/ksb/Module.pm index ed7ffaa..4b2c551 100644 --- a/modules/ksb/Module.pm +++ b/modules/ksb/Module.pm @@ -32,6 +32,8 @@ use ksb::BuildSystem::Qt4; use ksb::BuildSystem::KDE4; use ksb::BuildSystem::CMakeBootstrap; +use ksb::ModuleSet::Null; + use Storable 'dclone'; use Carp 'confess'; use Scalar::Util 'blessed'; @@ -93,8 +95,8 @@ sub phases sub moduleSet { my ($self) = @_; - return $self->{'module-set'} if exists $self->{'module-set'}; - return ''; + $self->{'module-set'} //= ksb::ModuleSet::Null->new(); + return $self->{'module-set'}; } sub setModuleSet diff --git a/modules/ksb/ModuleSet.pm b/modules/ksb/ModuleSet.pm new file mode 100644 index 0000000..7cf48dd --- /dev/null +++ b/modules/ksb/ModuleSet.pm @@ -0,0 +1,182 @@ +package ksb::ModuleSet; + +# Class: ModuleSet +# +# This represents a collective grouping of modules that share common options, +# and share a common repository (in this case, based on the git-repository-base +# option, but see also the more common ModuleSet::KDEProjects which is used for +# the special kde-projects repositories). +# +# This is parsed from module-set declarations in the rc-file. +# +# The major conceit here is several things: +# +# 1. A hash of options to set for each module read into this module set. +# 2. A list of module search declarations to be used to construct modules for +# this module set (in the case of kde-projects repository). For other +# repository types we can still consider it a 'search', but with the +# understanding that it's a 1:1 mapping to the 'found' module (which may not +# exist for real). +# 3. A list of module search declarations to *ignore* from this module set, +# using the same syntax as used to search for them in 2. This is only really +# useful at this point for kde-projects repository as everything else requires +# you to manually specify modules one-by-one (module-sets are only useful here +# for option grouping as per 1.). +# 4. A name, which must not be empty, although user-specified names cannot be +# assumed to be unique. +# 5. A ksb::PhaseList describing what phases of the build a module should +# participate in by default. +# +# See also: git-repository-base, ModuleSet::KDEProjects, use-modules + +use strict; +use warnings; +use v5.10; +no if $] >= 5.018, 'warnings', 'experimental::smartmatch'; + +our $VERSION = '0.10'; + +use ksb::Debug; +use ksb::Util; +use ksb::PhaseList; +use ksb::BuildContext; +use Storable qw(dclone); + +sub new +{ + my ($class, $ctx, $name) = @_; + $name //= ''; + + my $options = { + name => $name, + options => { }, + module_search_decls => [ ], + module_ignore_decls => [ ], + phase_list => ksb::PhaseList->new($ctx->phases()->phases()), + }; + + return bless $options, $class; +} + +sub name +{ + my $self = shift; + return $self->{name}; +} + +sub setName +{ + my ($self, $name) = @_; + $self->{name} = $name; + return; +} + +# Returns a deep-copied hashref, not a hash. +sub options +{ + my $self = shift; + return dclone($self->{options}); +} + +# Completely replaces stored options with the options given in the provided +# hashref. +sub setOptions +{ + my ($self, $hashref) = @_; + $self->{options} = $hashref; + return; +} + +# Just returns a reference to the existing ksb::PhaseList, there's no way to +# replace this, though you can alter the underlying phases through the +# ksb::PhaseList object itself. +sub phases +{ + my $self = shift; + return $self->{phase_list}; +} + +sub modulesToFind +{ + my $self = shift; + return @{$self->{module_search_decls}}; +} + +sub setModulesToFind +{ + my ($self, @moduleDecls) = @_; + $self->{module_search_decls} = [@moduleDecls]; + return; +} + +sub modulesToIgnore +{ + my $self = shift; + return @{$self->{module_ignore_decls}}; +} + +sub setModulesToIgnore +{ + my ($self, @moduleDecls) = @_; + $self->{module_ignore_decls} = [@moduleDecls]; + return; +} + +# Should be called for each new ksb::Module created in order to setup common +# module options. +sub _initializeNewModule +{ + my ($self, $newModule) = @_; + + $newModule->setModuleSet($self); + $newModule->setScmType('git'); + $newModule->phases->phases($self->phases()->phases()); + + # Dump all options into the existing ksb::Module's options. + $newModule->setOption(%{$self->options()}); +} + +# This function should be called after options are read and build metadata is +# available in order to convert this module set to a list of ksb::Module. +# Any modules ignored by this module set are excluded from the returned list. +# The modules returned have not been added to the build context. +sub convertToModules +{ + my ($self, $ctx) = @_; + + my @moduleList; # module names converted to ksb::Module objects. + my $optionsRef = $self->{options}; + + # Note: This returns a hashref, not a string. + my $repoSet = $ctx->getOption('git-repository-base'); + + # Setup default options for each module + # If we're in this method, we must be using the git-repository-base method + # of setting up a module-set, so there is no 'search' or 'ignore' to + # handle, just create ksb::Module and dump options into them. + for my $moduleItem ($self->modulesToFind()) { + my $moduleName = $moduleItem; + + $moduleName =~ s/\.git$//; + + my $newModule = ksb::Module->new($ctx, $moduleName); + + $self->_initializeNewModule($newModule); + + push @moduleList, $newModule; + + # Setup the only feature actually specific to a module-set, which is + # the repository handling. + my $selectedRepo = $repoSet->{$optionsRef->{'repository'}}; + $newModule->setOption('repository', $selectedRepo . $moduleItem); + } + + if (not scalar @moduleList) { + warning ("No modules were defined for the module-set $self->name()"); + warning ("You should use the g[b[use-modules] option to make the module-set useful."); + } + + return @moduleList; +} + +1; diff --git a/modules/ksb/ModuleSet/KDEProjects.pm b/modules/ksb/ModuleSet/KDEProjects.pm new file mode 100644 index 0000000..4a4b022 --- /dev/null +++ b/modules/ksb/ModuleSet/KDEProjects.pm @@ -0,0 +1,222 @@ +package ksb::ModuleSet::KDEProjects; + +# Class: ModuleSet::KDEProjects +# +# This represents a collective grouping of modules that share common options, +# and are obtained via the kde-projects database at +# https://projects.kde.org/kde_projects.xml +# +# See also the parent class ModuleSet, from which most functionality is derived. +# +# The only changes here are to allow for expanding out module specifications +# (except for ignored modules), by using KDEXMLReader. +# +# See also: ModuleSet + +use strict; +use warnings; +use v5.10; +no if $] >= 5.018, 'warnings', 'experimental::smartmatch'; + +our $VERSION = '0.10'; +our @ISA = qw(ksb::ModuleSet); + +use ksb::Module; +use ksb::Debug; +use ksb::KDEXMLReader; +use ksb::BuildContext 0.20; +use ksb::Util; + +# A 'new' subroutine is not needed, ksb::ModuleSet's should do the right thing + +# Simple utility subroutine. See List::Util's perldoc +sub none_true +{ + ($_ && return 0) for @_; + return 1; +} + +# Function: getMetadataModule +# +# A 'static' method that returns a that should be included first in +# the build context's module list. It will be configured to download required +# updates to the build-metadata required for kde-projects module support. +# It should only be included exactly once in the build context, if there are +# one or more ksb::ModuleSet::KDEProjects present in the module list. +# +# Parameters: +# ctx - the for this script execution. +# +# Returns: The to added to the beginning of the update. +sub getMetadataModule +{ + my $ctx = assert_isa(shift, 'ksb::BuildContext'); + + my $metadataModule = ksb::Module->new($ctx, 'kde-build-metadata'); + + # Hardcode the results instead of expanding out the project info + $metadataModule->setOption('repository', 'kde:kde-build-metadata'); + $metadataModule->setOption('#xml-full-path', 'kde-build-metadata'); + $metadataModule->setOption('#branch:stable', 'master'); + $metadataModule->setScmType('metadata'); + $metadataModule->setOption('disable-snapshots', 1); + $metadataModule->setOption('branch', 'master'); + + my $moduleSet = ksb::ModuleSet::KDEProjects->new($ctx, ''); + $metadataModule->setModuleSet($moduleSet); + + # Ensure we only ever try to update source, not build. + $metadataModule->phases()->phases('update'); + return $metadataModule; +} + +# Function: _expandModuleCandidates +# +# A class method which goes through the modules in our search list (assumed to +# be found in the kde-projects XML database) and expands them into their +# equivalent git modules, and returns the fully expanded list. Non kde-projects +# modules cause an error, as do modules that do not exist at all within the +# database. +# +# *Note*: Before calling this function, the kde-projects database itself must +# have been downloaded first. Additionally a handling build support +# metadata must be included at the beginning of the module list, see +# getMetadataModule() for details. +# +# *Note*: Any modules that are part of a module-set requiring a specific +# branch, that don't have that branch, are also elided with only a debug +# message. This allows for building older branches of KDE even when newer +# modules are eventually put into the database. +# +# Parameters: +# ctx - The in use. +# moduleSearchItem - The search description to expand in ksb::Modules. See +# _projectPathMatchesWildcardSearch for a description of the syntax. +# +# Returns: +# @modules - List of expanded git . +# +# Throws: +# Runtime - if the kde-projects database was required but couldn't be +# downloaded or read. +# Runtime - if the git-desired-protocol is unsupported. +# Runtime - if an "assumed" kde-projects module was not actually one. +sub _expandModuleCandidates +{ + my $self = assert_isa(shift, 'ksb::ModuleSet::KDEProjects'); + my $ctx = assert_isa(shift, 'ksb::BuildContext'); + my $moduleSearchItem = shift; + + my $databaseFile = $ctx->getKDEProjectMetadataFilehandle() or + croak_runtime("kde-projects repository information could not be downloaded: $!"); + my $srcdir = $ctx->getSourceDir(); + + my $protocol = $ctx->getOption('git-desired-protocol') || 'git'; + if (!list_has(['git', 'http'], $protocol)) { + error (" b[y[*] Invalid b[git-desired-protocol] $protocol"); + error (" b[y[*] Try setting this option to 'git' if you're not using a proxy"); + croak_runtime ("Invalid git-desired-protocol: $protocol"); + } + + my $xmlReader = ksb::KDEXMLReader->new($databaseFile); + my @allXmlResults = $xmlReader->getModulesForProject($moduleSearchItem, $protocol); + + # It's possible to match modules which are marked as inactive on + # projects.kde.org, elide those. + my @xmlResults = grep { $_->{'active'} ne 'false' } (@allXmlResults); + + # Bug 307694 + my $moduleSetBranch = $self->{'options'}->{'branch'} // ''; + if ($moduleSetBranch) { + debug ("Filtering kde-projects modules that don't have a $moduleSetBranch branch"); + @xmlResults = grep { + list_has($_->{'branches'}, $moduleSetBranch) + } (@xmlResults); + } + + if (!@xmlResults) { + warning (" y[b[*] Module y[$moduleSearchItem] is apparently XML-based, but contains no\n" . + "active modules to build!"); + my $count = scalar @allXmlResults; + if ($count > 0) { + warning ("\tAlthough no active modules are available, there were\n" . + "\t$count inactive modules. Perhaps the git modules are not ready?"); + } + } + + # Setup module options. + my @moduleList; + my @ignoreList = $self->modulesToIgnore(); + + foreach (@xmlResults) { + my $result = $_; + my $repo = $result->{'repo'}; + + # Prefer kde: alias to normal clone URL. + $repo =~ s(^git://anongit\.kde\.org/)(kde:); + + my $newModule = ksb::Module->new($ctx, $result->{'name'}); + $self->_initializeNewModule($newModule); + $newModule->setOption('repository', $repo); + $newModule->setOption('#xml-full-path', $result->{'fullName'}); + $newModule->setOption('#branch:stable', $result->{'branch:stable'}); + + my $tarball = $result->{'tarball'}; + $newModule->setOption('#snapshot-tarball', $tarball) if $tarball; + + if (none_true( + map { + ksb::KDEXMLReader::_projectPathMatchesWildcardSearch( + $result->{'fullName'}, + $_ + ) + } (@ignoreList))) + { + push @moduleList, $newModule; + } + else { + debug ("--- Ignoring matched active module $newModule in module set " . + $self->name()); + } + }; + + return @moduleList; +} + +# This function should be called after options are read and build metadata is +# available in order to convert this module set to a list of ksb::Module. +# Any modules ignored by this module set are excluded from the returned list. +# The modules returned have not been added to the build context. +sub convertToModules +{ + my ($self, $ctx) = @_; + + my @moduleList; # module names converted to ksb::Module objects. + + # Setup default options for each module + # Extraction of relevant XML modules will be handled immediately after + # this phase of execution. + for my $moduleItem ($self->modulesToFind()) { + # eval in case the XML processor throws an exception. + undef $@; + my @candidateModules = eval { + $self->_expandModuleCandidates($ctx, $moduleItem); + }; + + if ($@) { + die $@ if ref $@; # Forward exception objects up + croak_runtime("The XML for the KDE Project database could not be understood: $@"); + } + + push @moduleList, @candidateModules; + } + + if (not scalar @moduleList) { + warning ("No modules were defined for the module-set $self->name()"); + warning ("You should use the g[b[use-modules] option to make the module-set useful."); + } + + return @moduleList; +} + +1; diff --git a/modules/ksb/ModuleSet/Null.pm b/modules/ksb/ModuleSet/Null.pm new file mode 100644 index 0000000..1dd6b76 --- /dev/null +++ b/modules/ksb/ModuleSet/Null.pm @@ -0,0 +1,33 @@ +package ksb::ModuleSet::Null; + +# Class: ModuleSet::Null +# +# Used automatically by to represent the abscence of a without +# requiring definedness checks. + +use strict; +use warnings; +use v5.10; + +our $VERSION = '0.10'; +our @ISA = qw(ksb::ModuleSet); + +use ksb::Util; + +sub new +{ + my $class = shift; + return bless {}, $class; +} + +sub name +{ + return ''; +} + +sub convertToModules +{ + croak_internal("kdesrc-build should not have made it to this call. :-("); +} + +1;